diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml index 3f64fd392de..5a10146128a 100644 --- a/.github/workflows/manual-deploy-ext-tools.yml +++ b/.github/workflows/manual-deploy-ext-tools.yml @@ -47,4 +47,5 @@ jobs: needs: build with: cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }} - secrets: inherit \ No newline at end of file + secrets: inherit + diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml index c3ca5077a33..faedd27bebc 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merge.yml @@ -21,3 +21,4 @@ jobs: with: cli: python3 ./build/cloud/docker/docker-build.py delete --images base --tag ${{ github.head_ref }} secrets: inherit + diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index fe6350d59ca..b2b5105877b 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -86,3 +86,4 @@ jobs: commit: '${{ github.sha }}' fail_on_test_failures: true + diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java index 34b337731a5..069d7ae46a7 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/knockout/KnockoutAnalysis.java @@ -202,12 +202,12 @@ protected void run() throws Exception { if (family == null || StringUtils.isEmpty(family.getId())) { continue; } - List> trios = variantStorageManager.getTriosFromFamily(getStudy(), family, true, getToken()); - for (List trio : trios) { - String child = trio.get(2); + List trios = variantStorageManager.getTriosFromFamily(getStudy(), family, true, getToken()); + for (Trio trio : trios) { + String child = trio.getChild(); if (analysisParams.getSample().contains(child)) { - String father = trio.get(0); - String mother = trio.get(1); + String father = trio.getFather(); + String mother = trio.getMother(); triosMap.put(child, new Trio(family.getId(), "-".equals(father) ? null : father, "-".equals(mother) ? null : mother, diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java index 2d08fb77566..7a588c900f5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java @@ -1299,10 +1299,10 @@ public Integer getReleaseFilter(Query query, String sessionId) throws CatalogExc return release; } - public List> getTriosFromFamily( + public List getTriosFromFamily( String studyFqn, Family family, VariantStorageMetadataManager metadataManager, boolean skipIncompleteFamily, String sessionId) throws StorageEngineException, CatalogException { - List> trios = getTrios(studyFqn, metadataManager, family.getMembers(), sessionId); + List trios = getTrios(studyFqn, metadataManager, family.getMembers(), sessionId); if (trios.size() == 0) { if (skipIncompleteFamily) { logger.debug("Skip family '" + family.getId() + "'. "); @@ -1313,7 +1313,7 @@ public List> getTriosFromFamily( return trios; } - public List> getTriosFromSamples( + public List getTriosFromSamples( String studyFqn, VariantStorageMetadataManager metadataManager, Collection sampleIds, String token) throws CatalogException { OpenCGAResult individualResult = catalogManager.getIndividualManager() @@ -1330,12 +1330,12 @@ public List> getTriosFromSamples( return getTrios(studyFqn, metadataManager, individualResult.getResults(), token); } - public List> getTrios( + public List getTrios( String studyFqn, VariantStorageMetadataManager metadataManager, List membersList, String sessionId) throws CatalogException { int studyId = metadataManager.getStudyId(studyFqn); Map membersMap = membersList.stream().collect(Collectors.toMap(Individual::getUid, i -> i)); - List> trios = new LinkedList<>(); + List trios = new LinkedList<>(); for (Individual individual : membersList) { String fatherSample = null; String motherSample = null; @@ -1402,10 +1402,7 @@ public List> getTrios( // Allow one missing parent if (childSample != null && (fatherSample != null || motherSample != null)) { - trios.add(Arrays.asList( - fatherSample == null ? "-" : fatherSample, - motherSample == null ? "-" : motherSample, - childSample)); + trios.add(new Trio(fatherSample, motherSample, childSample)); } } return trios; diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java index 22b337e9621..bf0db6f34eb 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantStorageManager.java @@ -63,6 +63,7 @@ import org.opencb.opencga.core.models.operations.variant.*; import org.opencb.opencga.core.models.project.DataStore; import org.opencb.opencga.core.models.project.Project; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SamplePermissions; import org.opencb.opencga.core.models.study.Study; @@ -76,10 +77,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.VariantMetadataFactory; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.ProjectMetadata; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.BeaconResponse; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; @@ -397,7 +395,7 @@ public void sampleIndexAnnotate(String study, List samples, ObjectMap pa }); } - public DataResult> familyIndexUpdate(String study, + public DataResult familyIndexUpdate(String study, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { @@ -405,11 +403,11 @@ public DataResult> familyIndexUpdate(String study, }); } - public DataResult> familyIndex(String study, List familiesStr, boolean skipIncompleteFamilies, + public DataResult familyIndex(String study, List familiesStr, boolean skipIncompleteFamilies, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { - List> trios = new LinkedList<>(); + List trios = new LinkedList<>(); List events = new LinkedList<>(); VariantStorageMetadataManager metadataManager = engine.getMetadataManager(); VariantCatalogQueryUtils catalogUtils = new VariantCatalogQueryUtils(catalogManager); @@ -425,9 +423,9 @@ public DataResult> familyIndex(String study, List familiesS trios.addAll(catalogUtils.getTriosFromFamily(study, family, metadataManager, skipIncompleteFamilies, token)); } } - DataResult> dataResult = engine.familyIndex(study, trios, params); + DataResult dataResult = engine.familyIndex(study, trios, params); getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream() - .flatMap(Collection::stream) + .flatMap(t->t.toList().stream()) .collect(Collectors.toList()), token); return dataResult; }); @@ -439,7 +437,7 @@ private CatalogStorageMetadataSynchronizer getSynchronizer(VariantStorageEngine return synchronizer; } - public DataResult> familyIndexBySamples(String study, Collection samples, ObjectMap params, String token) + public DataResult familyIndexBySamples(String study, Collection samples, ObjectMap params, String token) throws CatalogException, StorageEngineException { return secureOperation(VariantFamilyIndexOperationTool.ID, study, params, token, engine -> { Collection thisSamples = samples; @@ -447,17 +445,16 @@ public DataResult> familyIndexBySamples(String study, Collection> trios = catalogUtils.getTriosFromSamples(study, engine.getMetadataManager(), thisSamples, token); - - DataResult> dataResult = engine.familyIndex(study, trios, params); + List trios = catalogUtils.getTriosFromSamples(study, engine.getMetadataManager(), thisSamples, token); + DataResult dataResult = engine.familyIndex(study, trios, params); getSynchronizer(engine).synchronizeCatalogSamplesFromStorage(study, trios.stream() - .flatMap(Collection::stream) + .flatMap(t -> t.toList().stream()) .collect(Collectors.toList()), token); return dataResult; }); } - public List> getTriosFromFamily(String study, Family family, boolean skipIncompleteFamilies, String token) + public List getTriosFromFamily(String study, Family family, boolean skipIncompleteFamilies, String token) throws CatalogException, StorageEngineException { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(study, token); return catalogUtils.getTriosFromFamily(study, family, variantStorageEngine.getMetadataManager(), skipIncompleteFamilies, token); @@ -1015,11 +1012,21 @@ protected VariantStorageEngine getVariantStorageEngineByProject(String project, return variantStorageEngine; } - private void setCellbaseConfiguration(VariantStorageEngine engine, String project, String token) + private void setCellbaseConfiguration(VariantStorageEngine engine, String projectId, String token) throws CatalogException { - CellBaseConfiguration cellbase = catalogManager.getProjectManager() - .get(project, new QueryOptions(INCLUDE, ProjectDBAdaptor.QueryParams.CELLBASE.key()), token) - .first().getCellbase(); + Project project = catalogManager.getProjectManager() + .get(projectId, new QueryOptions(INCLUDE, Arrays.asList( + ProjectDBAdaptor.QueryParams.CELLBASE.key(), + ProjectDBAdaptor.QueryParams.ORGANISM.key())), token) + .first(); + CellBaseConfiguration cellbase = project.getCellbase(); + ProjectOrganism organism = project.getOrganism(); + if (organism == null) { + throw new CatalogException("Missing organism in project '" + project.getFqn()+ "'"); + } else { + engine.getOptions().put(VariantStorageOptions.SPECIES.key(), organism.getScientificName()); + engine.getOptions().put(VariantStorageOptions.ASSEMBLY.key(), organism.getAssembly()); + } if (cellbase != null) { // To ensure that we use the project API key before using the storage API key if (StringUtils.isEmpty(cellbase.getApiKey()) && storageConfiguration.getCellbase() != null) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java index 48ff94151d7..94bfada60d5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java @@ -100,6 +100,7 @@ public class VariantFileIndexerOperationManager extends OperationManager { private List filesToIndex; private CatalogStorageMetadataSynchronizer synchronizer; private boolean fullSynchronize = false; + private boolean force; public VariantFileIndexerOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) { super(variantStorageManager, engine); @@ -138,6 +139,7 @@ private void check(String study, ObjectMap params, String token) throws Exceptio } resume = params.getBoolean(VariantStorageOptions.RESUME.key()); skipIndexedFiles = params.getBoolean(SKIP_INDEXED_FILES); + force = params.getBoolean(VariantStorageOptions.FORCE.key()); // Obtain the type of analysis (transform, load or index) step = getType(load, transform); @@ -589,6 +591,7 @@ private List filterTransformFiles(List fileList, boolean resume) thr break; case VariantIndexStatus.INDEXING: case VariantIndexStatus.TRANSFORMING: + case VariantIndexStatus.LOADING: if (resume) { filteredFiles.add(file); } else { @@ -603,14 +606,17 @@ private List filterTransformFiles(List fileList, boolean resume) thr } break; case VariantIndexStatus.TRANSFORMED: - case VariantIndexStatus.LOADING: case VariantIndexStatus.READY: default: String msg = "We can only " + step + " VCF files not transformed, the status is " + indexStatus; - if (skipIndexedFiles) { - logger.warn(msg); + if (force) { + filteredFiles.add(file); } else { - throw new StorageEngineException(msg); + if (skipIndexedFiles) { + logger.warn(msg); + } else { + throw new StorageEngineException(msg); + } } break; } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java index 4f6f7bb6452..3d97af2c238 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java @@ -508,6 +508,16 @@ protected boolean synchronizeFiles(StudyMetadata study, List files, String } fileSamplesMap.put(fileMetadata.getName(), samples); allSamples.addAll(fileMetadata.getSamples()); + if (samples.size() > 100) { + // Try to reuse value. + // If the file holds more than 100 samples, it's most likely this same set of samples is already present + for (Set value : fileSamplesMap.values()) { + if (value.equals(samples)) { + fileSamplesMap.put(fileMetadata.getName(), value); + break; + } + } + } } if (!indexedFilesFromStorage.isEmpty()) { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java index c542521329a..042784b844a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/mutationalSignature/MutationalSignatureLocalAnalysisExecutor.java @@ -592,7 +592,9 @@ private void computeSignatureFitting() throws IOException, ToolException, Catalo StringBuilder scriptParams = new StringBuilder("R CMD Rscript --vanilla ") .append("/opt/opencga/signature.tools.lib/scripts/signatureFit") .append(" --catalogues=/data/input/").append(cataloguesFile.getName()) - .append(" --outdir=/data/output"); + .append(" --outdir=/data/output") + .append(" --commonsigtier=T2"); + if (StringUtils.isNotEmpty(getFitMethod())) { scriptParams.append(" --fitmethod=").append(getFitMethod()); } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java index ed774beca11..c7258c79f74 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantFamilyIndexOperationTool.java @@ -24,6 +24,7 @@ import org.opencb.opencga.core.models.operations.variant.VariantFamilyIndexParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; +import org.opencb.opencga.storage.core.metadata.models.Trio; import java.util.Collections; import java.util.List; @@ -67,7 +68,7 @@ protected void check() throws Exception { @Override protected void run() throws Exception { step(() -> { - DataResult> trios; + DataResult trios; if (variantFamilyIndexParams.isUpdateIndex()) { trios = variantStorageManager.familyIndexUpdate(study, params, token); } else { diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java index c4ba731b647..cce9fc91e8a 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java @@ -90,6 +90,7 @@ protected void check() throws Exception { params.put(VariantStorageOptions.ANNOTATE.key(), indexParams.isAnnotate()); params.putIfNotEmpty(VariantStorageOptions.ANNOTATOR.key(), indexParams.getAnnotator()); params.put(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), indexParams.isOverwriteAnnotations()); + params.put(VariantStorageOptions.FORCE.key(), indexParams.isForceReload()); params.put(VariantStorageOptions.RESUME.key(), indexParams.isResume()); params.put(VariantStorageOptions.NORMALIZATION_SKIP.key(), indexParams.getNormalizationSkip()); params.putIfNotEmpty(VariantStorageOptions.NORMALIZATION_REFERENCE_GENOME.key(), indexParams.getReferenceGenome()); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java index 1954d7eb790..d4dd7ab8a89 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantSecondarySampleIndexOperationTool.java @@ -24,6 +24,7 @@ import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; +import org.opencb.opencga.storage.core.metadata.models.Trio; import java.util.ArrayList; import java.util.List; @@ -80,7 +81,7 @@ protected void run() throws Exception { } if (sampleIndexParams.isFamilyIndex()) { step("familyIndex", () -> { - DataResult> result = variantStorageManager.familyIndexBySamples(study, sampleIndexParams.getSample(), params, + DataResult result = variantStorageManager.familyIndexBySamples(study, sampleIndexParams.getSample(), params, getToken()); if (result.getEvents() != null) { for (Event event : result.getEvents()) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java index b855a379b08..71af17f701e 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java @@ -18,6 +18,7 @@ import org.junit.AfterClass; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; @@ -95,6 +96,7 @@ public AlignmentAnalysisTest(String storageEngine) { private CatalogManager catalogManager; private VariantStorageManager variantStorageManager; + @ClassRule public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource(); public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource(); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java index f45e8e77ec2..e528a8156bc 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java @@ -6,16 +6,20 @@ import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.core.NodeConfig; import org.junit.rules.ExternalResource; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; import org.opencb.commons.datastore.solr.SolrManager; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.storage.StorageConfiguration; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; public class RgaSolrExtenalResource extends ExternalResource { @@ -23,6 +27,7 @@ public class RgaSolrExtenalResource extends ExternalResource { private SolrClient solrClient; protected boolean embeded = true; + private Class testClass; public RgaSolrExtenalResource() { this(true); @@ -32,16 +37,23 @@ public RgaSolrExtenalResource(boolean embeded) { this.embeded = embeded; } + @Override + public Statement apply(Statement base, Description description) { + testClass = description.getTestClass(); + return super.apply(base, description); + } + @Override protected void before() throws Throwable { super.before(); - Path rootDir = getTmpRootDir(); + Path rootDir = Paths.get("target/test-data", "junit-rga-solr-" + TimeUtils.getTimeMillis()); + Files.createDirectories(rootDir); String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.getInstance().getBuildVersion(); String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.getInstance().getBuildVersion(); - copyConfigSetConfiguration(mainConfigSet, "managed-schema"); - copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema"); + copyConfigSetConfiguration(mainConfigSet, "managed-schema", rootDir); + copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema", rootDir); String solrHome = rootDir.resolve("solr").toString(); @@ -78,15 +90,15 @@ protected void after() { } } - private void copyConfigSetConfiguration(String configSet, String managedSchemaFile) throws IOException { + private void copyConfigSetConfiguration(String configSet, String managedSchemaFile, Path rootDir) throws IOException { // Copy configuration - getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml"); - getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema"); - getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json"); - getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt"); - getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt"); - getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt"); - getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt"); + getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml", rootDir); + getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema", rootDir); + getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json", rootDir); + getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt", rootDir); + getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt", rootDir); } public RgaEngine configure(StorageConfiguration storageConfiguration) { diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java index 31a59eebc2e..fafdb74d983 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/VariantAnalysisTest.java @@ -22,6 +22,7 @@ import org.hamcrest.CoreMatchers; import org.junit.*; import org.junit.experimental.categories.Category; +import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.opencb.biodata.models.clinical.Disorder; @@ -58,6 +59,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.cohort.Cohort; @@ -69,6 +71,8 @@ import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualInternal; import org.opencb.opencga.core.models.individual.Location; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleReferenceParam; @@ -80,6 +84,7 @@ import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.core.tools.result.ExecutionResultManager; import org.opencb.opencga.storage.core.StorageEngineFactory; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; @@ -126,6 +131,8 @@ public class VariantAnalysisTest { private static String cancer_sample = "AR2.10039966-01T"; private static String germline_sample = "AR2.10039966-01G"; + @Rule + public ExpectedException thrown = ExpectedException.none(); @Parameterized.Parameters(name = "{0}") public static Object[][] parameters() { @@ -1056,6 +1063,16 @@ public void testPedigreeGraph() throws CatalogException { assertEquals(base64, family.getPedigreeGraph().getBase64()); } + @Test + public void testCellbaseConfigure() throws Exception { + String project = "Project_test_cellbase_configure"; + catalogManager.getProjectManager().create(new ProjectCreateParams(project, project, "", "", "", new ProjectOrganism("hsapiens", "grch38"), null, null), QueryOptions.empty(), token); + + thrown.expect(StorageEngineException.class); + thrown.expectMessage("The storage engine is in mode=READ_ONLY"); + variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration("https://uk.ws.zettagenomics.com/cellbase/", "v5.2", "1", ""), false, null, token); + } + public void checkExecutionResult(ExecutionResult er) { checkExecutionResult(er, true); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java index 58aa48f7828..b38101ef933 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtilsTest.java @@ -60,6 +60,7 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.adaptors.VariantField; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; @@ -868,9 +869,9 @@ public void getTriosFromFamily() throws Exception { sampleMetadata -> sampleMetadata.setIndexStatus(TaskMetadata.Status.READY)); } - List> trios = queryUtils.getTriosFromFamily("s1", f1, metadataManager, true, sessionId); + List trios = queryUtils.getTriosFromFamily("s1", f1, metadataManager, true, sessionId); // System.out.println("trios = " + trios); - assertEquals(Arrays.asList(Arrays.asList("sample1", "sample2", "sample3"), Arrays.asList("sample1", "sample2", "sample4")), trios); + assertEquals(Arrays.asList(new Trio("sample1", "sample2", "sample3"), new Trio("sample1", "sample2", "sample4")), trios); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java index 38ef6f98a21..a5ec92aafec 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java @@ -36,20 +36,22 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.YesNoAuto; +import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.config.storage.StorageConfiguration; +import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortCreateParams; import org.opencb.opencga.core.models.common.IndexStatus; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.individual.Individual; -import org.opencb.opencga.core.models.individual.IndividualInternal; -import org.opencb.opencga.core.models.individual.Location; +import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.operations.variant.VariantAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondaryAnnotationIndexParams; import org.opencb.opencga.core.models.operations.variant.VariantSecondarySampleIndexParams; +import org.opencb.opencga.core.models.project.ProjectCreateParams; +import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.sample.*; import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.variant.VariantIndexParams; @@ -57,7 +59,9 @@ import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.LongTests; import org.opencb.opencga.core.tools.result.ExecutionResult; +import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata; +import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.core.variant.dummy.DummyVariantStorageEngine; @@ -75,6 +79,8 @@ import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.fail; @RunWith(Parameterized.class) @Category(LongTests.class) @@ -148,6 +154,16 @@ public void setUp() throws Throwable { @After public void tearDown() { if (hadoopExternalResource != null) { + + try { + VariantStorageEngine engine = opencga.getStorageEngineFactory().getVariantStorageEngine(storageEngine, DB_NAME); + if (storageEngine.equals(HadoopVariantStorageEngine.STORAGE_ENGINE_ID)) { + VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) engine.getDBAdaptor()), Paths.get(opencga.createTmpOutdir("_hbase_print_variants_AFTER")).toUri()); + } + } catch (Exception ignore) { + ignore.printStackTrace(); + } + hadoopExternalResource.after(); hadoopExternalResource = null; } @@ -246,7 +262,7 @@ private void loadDataset() throws Throwable { Collections.emptyList(), false, 0, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), IndividualInternal.init(), Collections.emptyMap()).setFather(individuals.get(0)).setMother(individuals.get(1)), Collections.singletonList(daughter), new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true), token).first()); catalogManager.getFamilyManager().create( STUDY, - new Family("f1", "f1", Collections.singletonList(phenotype), Collections.singletonList(disorder), null, null, 3, null, null), + new Family("f1", "f1", Collections.singletonList(phenotype), Collections.singletonList(disorder), null, null, 4, null, null), individuals.stream().map(Individual::getId).collect(Collectors.toList()), new QueryOptions(), token); @@ -277,6 +293,28 @@ public void setUpCatalogManager() throws Exception { } + @Test + public void testVariantFileReload() throws Exception { + try { + toolRunner.execute(VariantIndexOperationTool.class, STUDY, + new VariantIndexParams() + .setForceReload(false) + .setFile(file.getId()), + Paths.get(opencga.createTmpOutdir()), "index_reload", token); + fail("Should have thrown an exception"); + } catch (ToolException e) { + assertEquals(StorageEngineException.class, e.getCause().getClass()); + assertEquals("We can only INDEX VCF files not transformed, the status is READY", e.getCause().getMessage()); + } + + toolRunner.execute(VariantIndexOperationTool.class, STUDY, + new VariantIndexParams() + .setForceReload(true) + .setFile(file.getId()), + Paths.get(opencga.createTmpOutdir()), "index_reload", token); + + } + @Test public void testVariantSecondaryAnnotationIndex() throws Exception { @@ -375,6 +413,50 @@ public void testVariantSecondarySampleIndex() throws Exception { } } + @Test + public void testVariantSecondarySampleIndexPartialFamily() throws Exception { + Assume.assumeThat(storageEngine, anyOf( +// is(DummyVariantStorageEngine.STORAGE_ENGINE_ID), + is(HadoopVariantStorageEngine.STORAGE_ENGINE_ID) + )); + for (String sample : samples) { + SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); + assertEquals(sample, IndexStatus.READY, sampleIndex.getStatus().getId()); + assertEquals(sample, IndexStatus.NONE, sampleIndex.getFamilyStatus().getId()); + assertEquals(sample, 1, sampleIndex.getVersion().intValue()); + } + + Phenotype phenotype = new Phenotype("phenotype", "phenotype", ""); + Disorder disorder = new Disorder("disorder", "disorder", "", "", Collections.singletonList(phenotype), Collections.emptyMap()); + + catalogManager.getFamilyManager().delete(STUDY, Collections.singletonList("f1"), null, token); + catalogManager.getIndividualManager().update(STUDY, daughter, new IndividualUpdateParams() + .setMother(new IndividualReferenceParam(null, null)), null, token); + catalogManager.getFamilyManager().create( + STUDY, + new Family("f2", "f2", Collections.singletonList(phenotype), Collections.singletonList(disorder), null, null, 2, null, null), + Arrays.asList(father, daughter), new QueryOptions(), + token); + + // Run family index. The family index status should be READY on offspring + toolRunner.execute(VariantSecondarySampleIndexOperationTool.class, STUDY, + new VariantSecondarySampleIndexParams() + .setFamilyIndex(true) + .setSample(Arrays.asList(daughter)), + Paths.get(opencga.createTmpOutdir()), "index", token); + + for (String sample : samples) { + SampleInternalVariantSecondarySampleIndex sampleIndex = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondarySampleIndex(); + assertEquals(sample, IndexStatus.READY, sampleIndex.getStatus().getId()); + if (sample.equals(daughter)) { + assertEquals(sample, IndexStatus.READY, sampleIndex.getFamilyStatus().getId()); + } else { + assertEquals(sample, IndexStatus.NONE, sampleIndex.getFamilyStatus().getId()); + } + assertEquals(sample, 1, sampleIndex.getVersion().intValue()); + } + } + @Test public void testGwasIndex() throws Exception { // Variant scores can not be loaded in mongodb nor dummy @@ -408,6 +490,28 @@ public void testGwasIndex() throws Exception { }); } + @Test + public void testCellbaseConfigure() throws Exception { + String project = "Project_test_cellbase_configure"; + catalogManager.getProjectManager().create(new ProjectCreateParams(project, project, "", "", "", new ProjectOrganism("hsapiens", "GRCh38"), null, null), QueryOptions.empty(), token); + + CellBaseUtils cellBaseUtils = variantStorageManager.getVariantStorageEngineByProject(project, null, token).getCellBaseUtils(); + assertEquals(ParamConstants.CELLBASE_URL, cellBaseUtils.getURL()); + assertEquals(ParamConstants.CELLBASE_VERSION, cellBaseUtils.getVersion()); + assertEquals("hsapiens", cellBaseUtils.getSpecies()); + assertEquals("GRCh38", cellBaseUtils.getAssembly()); + + String newCellbase = "https://uk.ws.zettagenomics.com/cellbase/"; + String newCellbaseVersion = "v5.2"; + + assertNotEquals(newCellbase, cellBaseUtils.getURL()); + assertNotEquals(newCellbaseVersion, cellBaseUtils.getVersion()); + + variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, "1", ""), false, null, token); + CellBaseConfiguration cellbaseConfiguration = catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getCellbase(); +// assertTrue(family.getPedigreeGraph() != null); + } + public void checkExecutionResult(ExecutionResult er) { checkExecutionResult(er, true); } diff --git a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile index 5558259e3f5..ff1298c64d0 100644 --- a/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile +++ b/opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile @@ -17,14 +17,15 @@ RUN apt-get update -y && DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt ## Installation dependencies using R install.packages() is slower than apt-get but final size is 400GB smaller. R -e "install.packages(c('BiocManager', 'RCircos', 'nnls', 'ggplot2', 'jsonlite', 'optparse', 'knitr', 'configr', 'dplyr', 'rmarkdown', 'tidyr', 'httr', 'kinship2', 'limSolve'))" && \ R -e "BiocManager::install('BiocStyle')" && \ + R -e "BiocManager::install('BSgenome.Hsapiens.UCSC.hg38')" && \ ## signature.tools.lib installation \ - R -e 'install.packages(c("devtools", "getopt"), repos="https://www.stats.bris.ac.uk/R/")' && \ + R -e 'install.packages(c("devtools", "getopt"), repos=c("http://cran.rstudio.com/", "https://www.stats.bris.ac.uk/R/"))' && \ git clone https://github.com/Nik-Zainal-Group/signature.tools.lib.git /opt/opencga/signature.tools.lib WORKDIR /opt/opencga/signature.tools.lib RUN git fetch origin --tags && \ - git checkout tags/v2.4.1 && \ + git checkout tags/v2.4.2 && \ sed -i '/Mmusculus/d' DESCRIPTION && \ sed -i '/Cfamiliaris/d' DESCRIPTION && \ sed -i '/1000genomes/d' DESCRIPTION && \ diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java index 4bb045cb874..7d3f514a745 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java @@ -380,6 +380,7 @@ private void index() throws ToolException { cliOptions.genericVariantIndexOptions.family, cliOptions.genericVariantIndexOptions.somatic, cliOptions.genericVariantIndexOptions.load, + cliOptions.genericVariantIndexOptions.forceReload, cliOptions.genericVariantIndexOptions.loadSplitData, cliOptions.genericVariantIndexOptions.loadMultiFileData, cliOptions.genericVariantIndexOptions.loadSampleIndex, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index ee9f603f739..7639ed66faf 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2023-09-12 OpenCB +* Copyright 2015-2023-10-20 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index 832d5638d49..53be1d6dfbe 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -1,5 +1,5 @@ /* -* Copyright 2015-2023-09-12 OpenCB +* Copyright 2015-2023-10-20 OpenCB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java index 719d9999d96..39baa4b29e5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomFilesCommandExecutor.java @@ -17,7 +17,6 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.app.cli.session.SessionManager; -import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.client.config.ClientConfiguration; import org.opencb.opencga.client.rest.OpenCGAClient; import org.opencb.opencga.core.models.file.File; @@ -36,10 +35,10 @@ public CustomFilesCommandExecutor(ObjectMap options, String token, ClientConfigu super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse upload() throws Exception { + public RestResponse upload(CustomFilesCommandOptions.UploadCommandOptions commandOptions) throws Exception { // ObjectMap params = new ObjectMap() - options.append("fileFormat", ParamUtils.defaultString(String.valueOf(options.get("fileFormat")), File.Format.UNKNOWN.toString())) - .append("bioformat", ParamUtils.defaultString(String.valueOf(options.get("bioformat")), File.Bioformat.UNKNOWN.toString())); + options.append("fileFormat", options.getString("fileFormat", File.Format.UNKNOWN.toString())) + .append("bioformat", options.getString("bioformat", File.Bioformat.UNKNOWN.toString())); // //If the DEPRECATED parameter fileFormat has set we only override it if the new parameter format is also set // params.append("fileFormat", ParamUtils.defaultString(commandOptions.format, params.getString("fileFormat"))); // params.putIfNotEmpty("study", commandOptions.study); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java index 8d5742e09ec..dffedddcab4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomJobsCommandExecutor.java @@ -17,6 +17,8 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; +import org.opencb.opencga.app.cli.main.options.JobsCommandOptions; +import org.opencb.opencga.app.cli.main.utils.JobsLog; import org.opencb.opencga.app.cli.main.utils.JobsTopManager; import org.opencb.opencga.app.cli.session.SessionManager; import org.opencb.opencga.catalog.db.api.JobDBAdaptor; @@ -40,27 +42,26 @@ public CustomJobsCommandExecutor(ObjectMap options, String token, ClientConfigur super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse top() throws Exception { + public RestResponse top(CustomJobsCommandOptions.TopCommandOptions c) throws Exception { Query query = new Query(); - query.putIfNotEmpty(JobDBAdaptor.QueryParams.STUDY.key(), String.valueOf(options.get("study"))); - query.putIfNotEmpty(ParamConstants.JOB_TOOL_ID_PARAM, String.valueOf(options.get("toolId"))); - query.putIfNotEmpty(ParamConstants.INTERNAL_STATUS_PARAM, String.valueOf(options.get("internalStatus"))); - query.putIfNotEmpty(ParamConstants.JOB_USER_PARAM, String.valueOf(options.get("userId"))); - query.putIfNotEmpty(ParamConstants.JOB_PRIORITY_PARAM, String.valueOf(options.get("priority"))); - query.putAll(options); + query.putIfNotEmpty(JobDBAdaptor.QueryParams.STUDY.key(), c.study); + query.putIfNotEmpty(ParamConstants.JOB_TOOL_ID_PARAM, c.toolId); + query.putIfNotEmpty(ParamConstants.INTERNAL_STATUS_PARAM, c.internalStatus); + query.putIfNotEmpty(ParamConstants.JOB_USER_PARAM, c.userId); + query.putIfNotEmpty(ParamConstants.JOB_PRIORITY_PARAM, c.priority); + query.putAll(c.commonOptions.params); - new JobsTopManager(openCGAClient, query, 2, 20, 2, false).run(); + new JobsTopManager(openCGAClient, query, c.iterations, c.jobsLimit, c.delay, c.plain, c.columns).run(); RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); return res; } -/* + public RestResponse log(JobsCommandOptions.LogCommandOptions c) throws Exception { - // JobsCommandOptions.LogCommandOptions c = jobsCommandOptions.logCommandOptions; new JobsLog(openCGAClient, c, System.out).run(); RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); return res; } -*/ + } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java index 102a2b99255..a7ffdc3633b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomStudiesCommandExecutor.java @@ -51,29 +51,25 @@ public CustomStudiesCommandExecutor(ObjectMap options, String token, ClientConfi super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse runTemplates() throws Exception { + public RestResponse runTemplates(CustomStudiesCommandOptions.RunTemplatesCommandOptions c) throws Exception { logger.debug("Run template"); - // StudiesCommandOptions.RunTemplatesCommandOptions c = studiesCommandOptions.runTemplatesCommandOptions; - String study = getSingleValidStudy(String.valueOf(options.get("study"))); - TemplateParams templateParams = new TemplateParams(String.valueOf(options.get("id")), - Boolean.parseBoolean(String.valueOf(options.get("overwrite"))), - Boolean.parseBoolean(String.valueOf(options.get("resume")))); + String study = getSingleValidStudy(c.study); + TemplateParams templateParams = new TemplateParams(c.id, c.overwrite, c.resume); ObjectMap params = new ObjectMap(); return openCGAClient.getStudyClient().runTemplates(study, templateParams, params); } - public RestResponse uploadTemplates() throws Exception { + public RestResponse uploadTemplates(CustomStudiesCommandOptions.UploadTemplatesCommandOptions c) throws Exception { logger.debug("Upload template file"); - // StudiesCommandOptions.UploadTemplatesCommandOptions c = studiesCommandOptions.uploadTemplatesCommandOptions; ObjectMap params = new ObjectMap(); - String study = getSingleValidStudy(String.valueOf(options.get("study"))); - Path path = Paths.get(String.valueOf(options.get("inputFile"))); + String study = c.study = getSingleValidStudy(c.study); + Path path = Paths.get(c.file); if (!path.toFile().exists()) { - throw new CatalogException("File '" + options.get("inputFile") + "' not found"); + throw new CatalogException("File '" + path + "' not found"); } IOManagerFactory ioManagerFactory = new IOManagerFactory(); IOManager ioManager = ioManagerFactory.get(path.toUri()); @@ -106,10 +102,10 @@ public FileVisitResult visitFile(URI fileUri, BasicFileAttributes attrs) throws logger.debug("Compressing file in '" + manifestPath + "' before uploading"); ioManager.zip(fileList, manifestPath.toFile()); params.put("file", manifestPath.toString()); - } else if (String.valueOf(options.get("inputFile")).endsWith("zip")) { - params.put("file", String.valueOf(options.get("inputFile"))); + } else if (c.file.endsWith("zip")) { + params.put("file", c.file); } else { - throw new CatalogException("File '" + options.get("inputFile") + "' is not a zip file"); + throw new CatalogException("File '" + c.file + "' is not a zip file"); } RestResponse uploadResponse = openCGAClient.getStudyClient().uploadTemplates(study, params); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java index eab222ebc45..7cac620fdc3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/custom/CustomUsersCommandExecutor.java @@ -50,18 +50,19 @@ public CustomUsersCommandExecutor(ObjectMap options, String token, ClientConfigu super(options, token, clientConfiguration, session, appHome, logger, openCGAClient); } - public RestResponse login() throws Exception { + public RestResponse login(CustomUsersCommandOptions.LoginCommandOptions commandOptions) throws Exception { logger.debug("Login"); RestResponse res = new RestResponse<>(); try { - String user = String.valueOf(options.get("user")); - String password = String.valueOf(options.get("password")); + String user = commandOptions.user; + String password = commandOptions.password; if (StringUtils.isNotEmpty(user) && StringUtils.isNotEmpty(password)) { AuthenticationResponse response = null; try { response = openCGAClient.login(user, password); } catch (Exception e) { + logger.debug("Login error", e); Event event = new Event(); event.setMessage(e.getMessage()); event.setType(Event.Type.ERROR); @@ -73,16 +74,16 @@ public RestResponse login() throws Exception { res = session.saveSession(user, response, openCGAClient); println(getKeyValueAsFormattedString(LOGIN_OK, user)); } else { - String sessionId = String.valueOf(options.get("token")); + String token = session.getSession().getToken(); String errorMsg = "Missing password. "; - if (StringUtils.isNotEmpty(sessionId)) { + if (StringUtils.isNotEmpty(token)) { errorMsg += "Active token detected "; } CommandLineUtils.error(errorMsg); } } catch (Exception e) { CommandLineUtils.error(LOGIN_ERROR, e); - e.printStackTrace(); + logger.debug("Login error", e); Event event = new Event(); event.setMessage(LOGIN_ERROR + e.getMessage()); res.setType(QueryType.VOID); @@ -93,7 +94,7 @@ public RestResponse login() throws Exception { } - public RestResponse logout() throws IOException { + public RestResponse logout(CustomUsersCommandOptions.LogoutCommandOptions commandOptions) throws IOException { logger.debug("Logout"); RestResponse res = new RestResponse<>(); try { @@ -106,6 +107,7 @@ public RestResponse logout() throws IOException { res.setType(QueryType.VOID); } catch (Exception e) { CommandLineUtils.error("Logout fail", e); + logger.debug("Logout error", e); } return res; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java index a16efb4df47..19efed930b2 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AdminCommandExecutor.java @@ -1,25 +1,19 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import java.util.Map; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; import org.opencb.opencga.app.cli.main.options.AdminCommandOptions; - -import java.util.Map; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.admin.GroupSyncParams; import org.opencb.opencga.core.models.admin.InstallationParams; import org.opencb.opencga.core.models.admin.JWTParams; @@ -31,6 +25,8 @@ import org.opencb.opencga.core.models.study.Group; import org.opencb.opencga.core.models.user.Account; import org.opencb.opencga.core.models.user.User; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java index 328a83daeb4..c1e5a5e0b7c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisAlignmentCommandExecutor.java @@ -1,26 +1,20 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.AnalysisAlignmentCommandOptions; - +import java.util.List; import org.ga4gh.models.ReadAlignment; import org.opencb.biodata.models.alignment.GeneCoverageStats; import org.opencb.biodata.models.alignment.RegionCoverage; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.AnalysisAlignmentCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.alignment.AlignmentGeneCoverageStatsParams; import org.opencb.opencga.core.models.alignment.AlignmentIndexParams; import org.opencb.opencga.core.models.alignment.AlignmentQcParams; @@ -31,6 +25,8 @@ import org.opencb.opencga.core.models.alignment.PicardWrapperParams; import org.opencb.opencga.core.models.alignment.SamtoolsWrapperParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java index 3039ab1cc68..ff9f4431ca4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisClinicalCommandExecutor.java @@ -1,34 +1,27 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.AnalysisClinicalCommandOptions; - +import java.util.List; import org.opencb.biodata.models.clinical.ClinicalDiscussion; import org.opencb.biodata.models.clinical.ClinicalProperty; import org.opencb.biodata.models.clinical.interpretation.ClinicalVariant; import org.opencb.biodata.models.clinical.interpretation.InterpretationMethod; import org.opencb.commons.datastore.core.FacetField; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.AnalysisClinicalCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveReplaceAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.SaveInterpretationAs; import org.opencb.opencga.catalog.utils.ParamUtils.UpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByGeneSummary; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividualSummary; @@ -63,6 +56,8 @@ import org.opencb.opencga.core.models.study.configuration.ClinicalConsentAnnotationParam; import org.opencb.opencga.core.models.study.configuration.ClinicalConsentConfiguration; import org.opencb.opencga.core.models.study.configuration.InterpretationStudyConfiguration; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java index 601ea3739bf..2f670f539e8 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java @@ -1,23 +1,9 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.AnalysisVariantCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.ClinicalProperty.ModeOfInheritance; import org.opencb.biodata.models.clinical.ClinicalProperty.Penetrance; @@ -29,7 +15,15 @@ import org.opencb.biodata.models.variant.metadata.VariantMetadata; import org.opencb.biodata.models.variant.metadata.VariantSetStats; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.QueryResponse; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.AnalysisVariantCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByGene; import org.opencb.opencga.core.models.analysis.knockout.KnockoutByIndividual; import org.opencb.opencga.core.models.clinical.ExomiserWrapperParams; @@ -58,6 +52,8 @@ import org.opencb.opencga.core.models.variant.VariantExportParams; import org.opencb.opencga.core.models.variant.VariantIndexParams; import org.opencb.opencga.core.models.variant.VariantStatsAnalysisParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; import org.opencb.oskar.analysis.variant.gwas.GwasConfiguration; @@ -862,6 +858,7 @@ private RestResponse runIndex() throws Exception { putNestedIfNotNull(beanParams, "family",commandOptions.family, true); putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); putNestedIfNotNull(beanParams, "load",commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java index 92677f27fa4..54d3c8c3704 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/CohortsCommandExecutor.java @@ -1,28 +1,22 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.CohortsCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.CohortsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortAclEntryList; import org.opencb.opencga.core.models.cohort.CohortAclUpdateParams; @@ -33,6 +27,8 @@ import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java index 3e05c22aa5f..fcac8b23c88 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/DiseasePanelsCommandExecutor.java @@ -1,25 +1,19 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; import org.opencb.opencga.app.cli.main.options.DiseasePanelsCommandOptions; - -import org.opencb.biodata.models.clinical.interpretation.DiseasePanel; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.panel.PanelAclEntryList; @@ -27,6 +21,8 @@ import org.opencb.opencga.core.models.panel.PanelCreateParams; import org.opencb.opencga.core.models.panel.PanelImportParams; import org.opencb.opencga.core.models.panel.PanelUpdateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -210,7 +206,7 @@ private RestResponse distinct() throws Exception { } private RestResponse importPanels() throws Exception { - logger.debug("Executing import in Disease Panels command line"); + logger.debug("Executing importPanels in Disease Panels command line"); DiseasePanelsCommandOptions.ImportCommandOptions commandOptions = diseasePanelsCommandOptions.importCommandOptions; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java index 35e1596aba4..b7521c2c790 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FamiliesCommandExecutor.java @@ -1,28 +1,22 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.FamiliesCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.FamiliesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.family.Family; @@ -33,6 +27,8 @@ import org.opencb.opencga.core.models.family.FamilyQualityControl; import org.opencb.opencga.core.models.family.FamilyUpdateParams; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java index 039e0d536e3..62796417c62 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/FilesCommandExecutor.java @@ -1,66 +1,34 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.FilesCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; import java.io.DataInputStream; import java.io.InputStream; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.interpretation.Software; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandExecutor; +import org.opencb.opencga.app.cli.main.custom.CustomFilesCommandOptions; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.FilesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.alignment.AlignmentFileQualityControl; import org.opencb.opencga.core.models.alignment.CoverageFileQualityControl; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; +import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.File.Bioformat; import org.opencb.opencga.core.models.file.File.Format; -import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileAclEntryList; import org.opencb.opencga.core.models.file.FileAclUpdateParams; import org.opencb.opencga.core.models.file.FileContent; @@ -79,6 +47,8 @@ import org.opencb.opencga.core.models.file.SmallFileInternal; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.variant.VariantFileQualityControl; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -637,7 +607,7 @@ private RestResponse upload() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomFilesCommandExecutor customFilesCommandExecutor = new CustomFilesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customFilesCommandExecutor.upload(); + return customFilesCommandExecutor.upload(commandOptions); } private RestResponse acl() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java index 86fcc2f8d58..1516be90a6c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/IndividualsCommandExecutor.java @@ -1,32 +1,26 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.IndividualsCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.biodata.models.clinical.qc.SampleRelatednessReport; import org.opencb.biodata.models.core.OntologyTermAnnotation; import org.opencb.biodata.models.core.SexOntologyTermAnnotation; import org.opencb.biodata.models.pedigree.IndividualProperty; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.IndividualsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.StatusParams; import org.opencb.opencga.core.models.common.TsvAnnotationParams; import org.opencb.opencga.core.models.individual.Individual; @@ -39,6 +33,8 @@ import org.opencb.opencga.core.models.individual.IndividualUpdateParams; import org.opencb.opencga.core.models.individual.Location; import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index 93723bdc05e..17b78b206b7 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -1,41 +1,22 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; +import java.util.Date; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.JobsCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; +import org.opencb.opencga.app.cli.main.*; import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomJobsCommandExecutor; -import java.util.Date; -import org.opencb.commons.datastore.core.FacetField; -import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.JobsCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.FileContent; import org.opencb.opencga.core.models.job.Job; @@ -46,6 +27,8 @@ import org.opencb.opencga.core.models.job.JobTop; import org.opencb.opencga.core.models.job.JobUpdateParams; import org.opencb.opencga.core.models.job.ToolInfo; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.result.ExecutionResult; import org.opencb.opencga.core.tools.result.ExecutorInfo; @@ -123,6 +106,12 @@ public void execute() throws Exception { case "log-tail": queryResponse = tailLog(); break; + case "log": + CustomJobsCommandExecutor customJobsCommandExecutor = new CustomJobsCommandExecutor( + new ObjectMap(jobsCommandOptions.commonCommandOptions.params), token, + clientConfiguration, getSessionManager(), appHome, getLogger()); + queryResponse = customJobsCommandExecutor.log(jobsCommandOptions.logCommandOptions); + break; default: logger.error("Subcommand not valid"); break; @@ -364,7 +353,7 @@ private RestResponse top() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomJobsCommandExecutor customJobsCommandExecutor = new CustomJobsCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customJobsCommandExecutor.top(); + return customJobsCommandExecutor.top(commandOptions); } private RestResponse acl() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java index 3f8448898be..e1d64ae2d4a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/MetaCommandExecutor.java @@ -1,25 +1,20 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.MetaCommandOptions; - import java.util.List; import java.util.Map; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.MetaCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java index 78ed32b92dc..494067498b1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java @@ -1,25 +1,18 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.OperationsVariantStorageCommandOptions; - +import java.util.List; import org.opencb.biodata.models.variant.metadata.Aggregation; import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.OperationsVariantStorageCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.YesNoAuto; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; @@ -44,6 +37,8 @@ import org.opencb.opencga.core.models.variant.VariantSampleDeleteParams; import org.opencb.opencga.core.models.variant.VariantStorageMetadataSynchronizeParams; import org.opencb.opencga.core.models.variant.VariantStudyDeleteParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -530,6 +525,7 @@ private RestResponse indexVariant() throws Exception { putNestedIfNotNull(beanParams, "family",commandOptions.family, true); putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true); putNestedIfNotNull(beanParams, "load",commandOptions.load, true); + putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true); putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true); putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true); putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true); @@ -600,6 +596,7 @@ private RestResponse launcherVariantIndex() throws Exception { putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true); putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true); putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true); + putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true); putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true); putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true); putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java index 2ed3593e612..e1757e59f09 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ProjectsCommandExecutor.java @@ -1,30 +1,26 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; +import java.util.List; +import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.utils.PrintUtils; - +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; import org.opencb.opencga.app.cli.main.options.ProjectsCommandOptions; - -import org.opencb.commons.datastore.core.FacetField; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.project.ProjectCreateParams; import org.opencb.opencga.core.models.project.ProjectOrganism; import org.opencb.opencga.core.models.project.ProjectUpdateParams; import org.opencb.opencga.core.models.study.Study; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java index a88c8330a26..e3bd0666741 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/SamplesCommandExecutor.java @@ -1,29 +1,23 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.SamplesCommandOptions; - +import java.util.List; import java.util.Map; import org.opencb.biodata.models.core.OntologyTermAnnotation; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.SamplesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.ParamUtils.CompleteUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.ExternalSource; import org.opencb.opencga.core.models.common.RgaIndex.Status; import org.opencb.opencga.core.models.common.StatusParams; @@ -38,6 +32,8 @@ import org.opencb.opencga.core.models.sample.SampleQualityControl; import org.opencb.opencga.core.models.sample.SampleUpdateParams; import org.opencb.opencga.core.models.sample.SampleVariantQualityControlMetrics; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java index 944262e77f1..e183fcae20d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/StudiesCommandExecutor.java @@ -1,54 +1,29 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.StudiesCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; import java.io.InputStream; import java.lang.Object; import java.net.URL; +import java.util.HashMap; +import java.util.List; import java.util.Set; import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandExecutor; +import org.opencb.opencga.app.cli.main.custom.CustomStudiesCommandOptions; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.StudiesCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveForceRemoveAction; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.audit.AuditRecord.Status.Result; import org.opencb.opencga.core.models.audit.AuditRecord; @@ -73,6 +48,8 @@ import org.opencb.opencga.core.models.study.Variable; import org.opencb.opencga.core.models.study.VariableSet; import org.opencb.opencga.core.models.study.VariableSetCreateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -464,7 +441,7 @@ private RestResponse runTemplates() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomStudiesCommandExecutor customStudiesCommandExecutor = new CustomStudiesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customStudiesCommandExecutor.runTemplates(); + return customStudiesCommandExecutor.runTemplates(commandOptions); } private RestResponse uploadTemplates() throws Exception { @@ -478,7 +455,7 @@ private RestResponse uploadTemplates() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } CustomStudiesCommandExecutor customStudiesCommandExecutor = new CustomStudiesCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customStudiesCommandExecutor.uploadTemplates(); + return customStudiesCommandExecutor.uploadTemplates(commandOptions); } private RestResponse deleteTemplates() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java index b2cca1037ba..0c19edfe709 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java @@ -1,39 +1,23 @@ package org.opencb.opencga.app.cli.main.executors; import com.fasterxml.jackson.databind.DeserializationFeature; -import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.*; -import org.opencb.opencga.core.response.RestResponse; -import org.opencb.opencga.client.exceptions.ClientException; -import org.opencb.commons.datastore.core.ObjectMap; - -import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; -import org.opencb.opencga.core.common.JacksonUtils; - import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; import java.util.HashMap; -import org.opencb.opencga.core.response.QueryType; -import org.opencb.commons.utils.PrintUtils; - -import org.opencb.opencga.app.cli.main.options.UsersCommandOptions; - -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandOptions; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; -import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; +import java.util.List; import java.util.Map; +import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.utils.PrintUtils; +import org.opencb.opencga.app.cli.main.*; +import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandExecutor; +import org.opencb.opencga.app.cli.main.custom.CustomUsersCommandOptions; +import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; +import org.opencb.opencga.app.cli.main.options.UsersCommandOptions; +import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction; +import org.opencb.opencga.client.exceptions.ClientException; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.user.AuthenticationResponse; @@ -44,6 +28,8 @@ import org.opencb.opencga.core.models.user.User; import org.opencb.opencga.core.models.user.UserFilter; import org.opencb.opencga.core.models.user.UserUpdateParams; +import org.opencb.opencga.core.response.QueryType; +import org.opencb.opencga.core.response.RestResponse; /* @@ -105,6 +91,12 @@ public void execute() throws Exception { case "update": queryResponse = update(); break; + case "logout": + CustomUsersCommandExecutor customUsersCommandExecutor = new CustomUsersCommandExecutor( + new ObjectMap(usersCommandOptions.commonCommandOptions.params), token, + clientConfiguration, getSessionManager(), appHome, getLogger()); + queryResponse = customUsersCommandExecutor.logout(usersCommandOptions.logoutCommandOptions); + break; default: logger.error("Subcommand not valid"); break; @@ -123,7 +115,7 @@ private RestResponse login() throws Exception { queryParams.putIfNotEmpty("password", commandOptions.password); queryParams.putIfNotEmpty("refreshToken", commandOptions.refreshToken); CustomUsersCommandExecutor customUsersCommandExecutor = new CustomUsersCommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger()); - return customUsersCommandExecutor.login(); + return customUsersCommandExecutor.login(commandOptions); } private RestResponse password() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java index 0b37bec1c1c..c69a1700845 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AnalysisVariantCommandOptions.java @@ -1055,91 +1055,94 @@ public class RunIndexCommandOptions { @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) public String jobTags; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--outdir"}, description = "Output directory", required = false, arity = 1) public String outdir; - @Parameter(names = {"--transform"}, description = "The body web service transform parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean transform = false; - @Parameter(names = {"--gvcf"}, description = "The body web service gvcf parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean gvcf = false; - @Parameter(names = {"--normalization-skip"}, description = "The body web service normalizationSkip parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean normalizationSkip = false; - @Parameter(names = {"--reference-genome"}, description = "The body web service referenceGenome parameter", required = false, arity = 1) + @Parameter(names = {"--reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String referenceGenome; - @Parameter(names = {"--fail-on-malformed-lines"}, description = "The body web service failOnMalformedLines parameter", required = false, arity = 1) + @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean family = false; - @Parameter(names = {"--somatic"}, description = "The body web service somatic parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean somatic = false; - @Parameter(names = {"--load"}, description = "The body web service load parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean load = false; - @Parameter(names = {"--load-split-data"}, description = "The body web service loadSplitData parameter", required = false, arity = 1) + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) + public boolean forceReload = false; + + @Parameter(names = {"--load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String loadSplitData; - @Parameter(names = {"--load-multi-file-data"}, description = "The body web service loadMultiFileData parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean loadMultiFileData = false; - @Parameter(names = {"--load-sample-index"}, description = "The body web service loadSampleIndex parameter", required = false, arity = 1) + @Parameter(names = {"--load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String loadSampleIndex; - @Parameter(names = {"--load-archive"}, description = "The body web service loadArchive parameter", required = false, arity = 1) + @Parameter(names = {"--load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String loadArchive; - @Parameter(names = {"--load-hom-ref"}, description = "The body web service loadHomRef parameter", required = false, arity = 1) + @Parameter(names = {"--load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String loadHomRef; - @Parameter(names = {"--post-load-check"}, description = "The body web service postLoadCheck parameter", required = false, arity = 1) + @Parameter(names = {"--post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String postLoadCheck; - @Parameter(names = {"--include-genotypes"}, description = "The body web service includeGenotypes parameter", required = false, arity = 1) + @Parameter(names = {"--include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String includeGenotypes; - @Parameter(names = {"--include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) + @Parameter(names = {"--include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String includeSampleData; - @Parameter(names = {"--merge"}, description = "The body web service merge parameter", required = false, arity = 1) + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String merge; - @Parameter(names = {"--deduplication-policy"}, description = "The body web service deduplicationPolicy parameter", required = false, arity = 1) + @Parameter(names = {"--deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String deduplicationPolicy; - @Parameter(names = {"--calculate-stats"}, description = "The body web service calculateStats parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean calculateStats = false; - @Parameter(names = {"--aggregated"}, description = "The body web service aggregated parameter", required = false, arity = 1) + @Parameter(names = {"--aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String aggregated; - @Parameter(names = {"--aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) + @Parameter(names = {"--aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String aggregationMappingFile; - @Parameter(names = {"--annotate"}, description = "The body web service annotate parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean annotate = false; - @Parameter(names = {"--annotator"}, description = "The body web service annotator parameter", required = false, arity = 1) + @Parameter(names = {"--annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String annotator; - @Parameter(names = {"--overwrite-annotations"}, description = "The body web service overwriteAnnotations parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean overwriteAnnotations = false; - @Parameter(names = {"--index-search"}, description = "The body web service indexSearch parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexSearch = false; - @Parameter(names = {"--skip-indexed-files"}, description = "The body web service skipIndexedFiles parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean skipIndexedFiles = false; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java index d0b8a26d019..a9b95be294a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/OperationsVariantStorageCommandOptions.java @@ -449,91 +449,94 @@ public class IndexVariantCommandOptions { @Parameter(names = {"--study", "-s"}, description = "Study [[user@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--file"}, description = "List of files to be indexed.", required = false, arity = 1) public String file; - @Parameter(names = {"--resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean resume = false; - @Parameter(names = {"--outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--outdir"}, description = "Output directory", required = false, arity = 1) public String outdir; - @Parameter(names = {"--transform"}, description = "The body web service transform parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean transform = false; - @Parameter(names = {"--gvcf"}, description = "The body web service gvcf parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean gvcf = false; - @Parameter(names = {"--normalization-skip"}, description = "The body web service normalizationSkip parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean normalizationSkip = false; - @Parameter(names = {"--reference-genome"}, description = "The body web service referenceGenome parameter", required = false, arity = 1) + @Parameter(names = {"--reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String referenceGenome; - @Parameter(names = {"--fail-on-malformed-lines"}, description = "The body web service failOnMalformedLines parameter", required = false, arity = 1) + @Parameter(names = {"--fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String failOnMalformedLines; - @Parameter(names = {"--family"}, description = "The body web service family parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean family = false; - @Parameter(names = {"--somatic"}, description = "The body web service somatic parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean somatic = false; - @Parameter(names = {"--load"}, description = "The body web service load parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean load = false; - @Parameter(names = {"--load-split-data"}, description = "The body web service loadSplitData parameter", required = false, arity = 1) + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) + public boolean forceReload = false; + + @Parameter(names = {"--load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String loadSplitData; - @Parameter(names = {"--load-multi-file-data"}, description = "The body web service loadMultiFileData parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean loadMultiFileData = false; - @Parameter(names = {"--load-sample-index"}, description = "The body web service loadSampleIndex parameter", required = false, arity = 1) + @Parameter(names = {"--load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String loadSampleIndex; - @Parameter(names = {"--load-archive"}, description = "The body web service loadArchive parameter", required = false, arity = 1) + @Parameter(names = {"--load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String loadArchive; - @Parameter(names = {"--load-hom-ref"}, description = "The body web service loadHomRef parameter", required = false, arity = 1) + @Parameter(names = {"--load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String loadHomRef; - @Parameter(names = {"--post-load-check"}, description = "The body web service postLoadCheck parameter", required = false, arity = 1) + @Parameter(names = {"--post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String postLoadCheck; - @Parameter(names = {"--include-genotypes"}, description = "The body web service includeGenotypes parameter", required = false, arity = 1) + @Parameter(names = {"--include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String includeGenotypes; - @Parameter(names = {"--include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) + @Parameter(names = {"--include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String includeSampleData; - @Parameter(names = {"--merge"}, description = "The body web service merge parameter", required = false, arity = 1) + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String merge; - @Parameter(names = {"--deduplication-policy"}, description = "The body web service deduplicationPolicy parameter", required = false, arity = 1) + @Parameter(names = {"--deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String deduplicationPolicy; - @Parameter(names = {"--calculate-stats"}, description = "The body web service calculateStats parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean calculateStats = false; - @Parameter(names = {"--aggregated"}, description = "Enum param allowed values: NONE, BASIC, EVS, EXAC", required = false, arity = 1) + @Parameter(names = {"--aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String aggregated; - @Parameter(names = {"--aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) + @Parameter(names = {"--aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String aggregationMappingFile; - @Parameter(names = {"--annotate"}, description = "The body web service annotate parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean annotate = false; - @Parameter(names = {"--annotator"}, description = "The body web service annotator parameter", required = false, arity = 1) + @Parameter(names = {"--annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String annotator; - @Parameter(names = {"--overwrite-annotations"}, description = "The body web service overwriteAnnotations parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean overwriteAnnotations = false; - @Parameter(names = {"--index-search"}, description = "The body web service indexSearch parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexSearch = false; - @Parameter(names = {"--skip-indexed-files"}, description = "The body web service skipIndexedFiles parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean skipIndexedFiles = false; } @@ -580,91 +583,94 @@ public class LauncherVariantIndexCommandOptions { @Parameter(names = {"--max-jobs"}, description = "The body web service maxJobs parameter", required = false, arity = 1) public Integer maxJobs; - @Parameter(names = {"--index-params-file"}, description = "The body web service file parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-file"}, description = "List of files to be indexed.", required = false, arity = 1) public String indexParamsFile; - @Parameter(names = {"--index-params-resume"}, description = "The body web service resume parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-resume"}, description = "Resume a previously failed index operation", required = false, help = true, arity = 0) public boolean indexParamsResume = false; - @Parameter(names = {"--index-params-outdir"}, description = "The body web service outdir parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-outdir"}, description = "Output directory", required = false, arity = 1) public String indexParamsOutdir; - @Parameter(names = {"--index-params-transform"}, description = "The body web service transform parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-transform"}, description = "If present it only runs the transform stage, no load is executed", required = false, help = true, arity = 0) public boolean indexParamsTransform = false; - @Parameter(names = {"--index-params-gvcf"}, description = "The body web service gvcf parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-gvcf"}, description = "Hint to indicate that the input file is in gVCF format.", required = false, help = true, arity = 0) public boolean indexParamsGvcf = false; - @Parameter(names = {"--index-params-normalization-skip"}, description = "The body web service normalizationSkip parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-normalization-skip"}, description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base", required = false, help = true, arity = 0) public boolean indexParamsNormalizationSkip = false; - @Parameter(names = {"--index-params-reference-genome"}, description = "The body web service referenceGenome parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-reference-genome"}, description = "Reference genome in FASTA format used during the normalization step for a complete left alignment", required = false, arity = 1) public String indexParamsReferenceGenome; - @Parameter(names = {"--index-params-fail-on-malformed-lines"}, description = "The body web service failOnMalformedLines parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-fail-on-malformed-lines"}, description = "Fail when encountering malformed lines. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsFailOnMalformedLines; - @Parameter(names = {"--index-params-family"}, description = "The body web service family parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-family"}, description = "Indicate that the files to be loaded are part of a family. This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards", required = false, help = true, arity = 0) public boolean indexParamsFamily = false; - @Parameter(names = {"--index-params-somatic"}, description = "The body web service somatic parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-somatic"}, description = "Indicate that the files to be loaded contain somatic samples. This will set 'load-hom-ref' to YES if it was in AUTO.", required = false, help = true, arity = 0) public boolean indexParamsSomatic = false; - @Parameter(names = {"--index-params-load"}, description = "The body web service load parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-load"}, description = "If present only the load stage is executed, transformation is skipped", required = false, help = true, arity = 0) public boolean indexParamsLoad = false; - @Parameter(names = {"--index-params-load-split-data"}, description = "The body web service loadSplitData parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-force-reload"}, description = "If the file is already loaded, force a file reload", required = false, help = true, arity = 0) + public boolean indexParamsForceReload = false; + + @Parameter(names = {"--index-params-load-split-data"}, description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.", required = false, arity = 1) public String indexParamsLoadSplitData; - @Parameter(names = {"--index-params-load-multi-file-data"}, description = "The body web service loadMultiFileData parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-load-multi-file-data"}, description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.", required = false, help = true, arity = 0) public boolean indexParamsLoadMultiFileData = false; - @Parameter(names = {"--index-params-load-sample-index"}, description = "The body web service loadSampleIndex parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-load-sample-index"}, description = "Build sample index while loading. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsLoadSampleIndex; - @Parameter(names = {"--index-params-load-archive"}, description = "The body web service loadArchive parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-load-archive"}, description = "Load archive data. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsLoadArchive; - @Parameter(names = {"--index-params-load-hom-ref"}, description = "The body web service loadHomRef parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-load-hom-ref"}, description = "Load HOM_REF genotypes. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsLoadHomRef; - @Parameter(names = {"--index-params-post-load-check"}, description = "The body web service postLoadCheck parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-post-load-check"}, description = "Execute post load checks over the database. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsPostLoadCheck; - @Parameter(names = {"--index-params-include-genotypes"}, description = "The body web service includeGenotypes parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-include-genotypes"}, description = "Load the genotype data for the current file. This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. Use this parameter skip load data when the GT field is not reliable, or its only value across the file is './.'. If 'auto', genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. (yes, no, auto) [auto]", required = false, arity = 1) public String indexParamsIncludeGenotypes; - @Parameter(names = {"--index-params-include-sample-data"}, description = "The body web service includeSampleData parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-include-sample-data"}, description = "Index including other sample data fields (i.e. FORMAT fields). Use 'all', 'none', or CSV with the fields to load.", required = false, arity = 1) public String indexParamsIncludeSampleData; - @Parameter(names = {"--index-params-merge"}, description = "The body web service merge parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-merge"}, description = "Currently two levels of merge are supported: 'basic' mode merge genotypes of the same variants while 'advanced' merge multiallelic and overlapping variants.", required = false, arity = 1) public String indexParamsMerge; - @Parameter(names = {"--index-params-deduplication-policy"}, description = "The body web service deduplicationPolicy parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-deduplication-policy"}, description = "Specify how duplicated variants should be handled. Available policies: 'discard', 'maxQual'", required = false, arity = 1) public String indexParamsDeduplicationPolicy; - @Parameter(names = {"--index-params-calculate-stats"}, description = "The body web service calculateStats parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-calculate-stats"}, description = "Calculate indexed variants statistics after the load step", required = false, help = true, arity = 0) public boolean indexParamsCalculateStats = false; - @Parameter(names = {"--index-params-aggregated"}, description = "Enum param allowed values: NONE, BASIC, EVS, EXAC", required = false, arity = 1) + @Parameter(names = {"--index-params-aggregated"}, description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC", required = false, arity = 1) public String indexParamsAggregated; - @Parameter(names = {"--index-params-aggregation-mapping-file"}, description = "The body web service aggregationMappingFile parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-aggregation-mapping-file"}, description = "File containing population names mapping in an aggregated VCF file", required = false, arity = 1) public String indexParamsAggregationMappingFile; - @Parameter(names = {"--index-params-annotate"}, description = "The body web service annotate parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-annotate"}, description = "Annotate indexed variants after the load step", required = false, help = true, arity = 0) public boolean indexParamsAnnotate = false; - @Parameter(names = {"--index-params-annotator"}, description = "The body web service annotator parameter", required = false, arity = 1) + @Parameter(names = {"--index-params-annotator"}, description = "Annotation source {cellbase_rest, cellbase_db_adaptor}", required = false, arity = 1) public String indexParamsAnnotator; - @Parameter(names = {"--index-params-overwrite-annotations"}, description = "The body web service overwriteAnnotations parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-overwrite-annotations"}, description = "Overwrite annotations in variants already present", required = false, help = true, arity = 0) public boolean indexParamsOverwriteAnnotations = false; - @Parameter(names = {"--index-params-index-search"}, description = "The body web service indexSearch parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-index-search"}, description = "Add files to the secondary search index", required = false, help = true, arity = 0) public boolean indexParamsIndexSearch = false; - @Parameter(names = {"--index-params-skip-indexed-files"}, description = "The body web service skipIndexedFiles parameter", required = false, help = true, arity = 0) + @Parameter(names = {"--index-params-skip-indexed-files"}, description = "Do not fail if any of the input files was already indexed.", required = false, help = true, arity = 0) public boolean indexParamsSkipIndexedFiles = false; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java index 418f6ed3152..6ff2cc5bff5 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_0/catalog/CalculatePedigreeGraphMigration.java @@ -34,14 +34,28 @@ public class CalculatePedigreeGraphMigration extends MigrationTool { protected void run() throws Exception { MigrationRun migrationRun = getMigrationRun(); + // Map study studyFqn -> job Map jobs = new HashMap<>(); for (JobReferenceParam jobReference : migrationRun.getJobs()) { Job job = catalogManager.getJobManager().get(jobReference.getStudyId(), jobReference.getId(), new QueryOptions(), token) .first(); - logger.info("Registering job {} for study {} to migrate", job.getId(), job.getStudy().getId()); + logger.info("Reading already executed job '{}' for study '{}' with status '{}'", + job.getId(), + job.getStudy().getId(), + job.getInternal().getStatus().getId()); jobs.put(job.getStudy().getId(), job); } - for (String study : getStudies()) { + + Set studies = new LinkedHashSet<>(getStudies()); + logger.info("Study IDs (num. total = {}) to initialize pedigree graphs: {}", studies.size(), StringUtils.join(studies, ", ")); + + // Ensure that studies with already executed jobs are included in the migration run + getMigrationRun().getJobs().forEach(j -> studies.add(j.getStudyId())); + + logger.info("Study IDs (num. total = {}) after adding studies from migration jobs: {}", studies.size(), + StringUtils.join(studies, ", ")); + + for (String study : studies) { Job job = jobs.get(study); if (job != null) { String status = job.getInternal().getStatus().getId(); @@ -61,8 +75,9 @@ protected void run() throws Exception { logger.info("Adding new job to migrate/initialize pedigree graph for study {}", study); ObjectMap params = new ObjectMap() .append(ParamConstants.STUDY_PARAM, study); - getMigrationRun().addJob(catalogManager.getJobManager().submit(study, PedigreeGraphInitAnalysis.ID, Enums.Priority.MEDIUM, - params, null, null, null, new ArrayList<>(), token).first()); + Job newJob = catalogManager.getJobManager().submit(study, PedigreeGraphInitAnalysis.ID, Enums.Priority.MEDIUM, + params, null, null, null, new ArrayList<>(), token).first(); + getMigrationRun().addJob(newJob); } } @@ -73,7 +88,7 @@ public List getStudies() throws CatalogException { for (Project project : catalogManager.getProjectManager().search(new Query(), projectOptions, token).getResults()) { if (CollectionUtils.isNotEmpty(project.getStudies())) { for (Study study : project.getStudies()) { - String id = project.getId() + ":" + study.getId(); + String id = study.getFqn(); for (Family family : catalogManager.getFamilyManager().search(id, new Query(), familyOptions, token).getResults()) { if (PedigreeGraphUtils.hasMinTwoGenerations(family) && (family.getPedigreeGraph() == null || StringUtils.isEmpty(family.getPedigreeGraph().getBase64()))) { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java new file mode 100644 index 00000000000..593eb6a15a0 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_8_6/storage/MarkUnknownLargestVariantLength.java @@ -0,0 +1,43 @@ +package org.opencb.opencga.app.migrations.v2_8_6.storage; + +import org.opencb.opencga.app.migrations.StorageMigrationTool; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; +import org.opencb.opencga.storage.core.variant.VariantStorageEngine; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; + +import java.util.Map; + +@Migration(id = "mark_unknown_largest_variant_length" , + description = "Mark as unknown largest variant length", + version = "2.8.6", + domain = Migration.MigrationDomain.STORAGE, + language = Migration.MigrationLanguage.JAVA, + patch = 1, + date = 20230927 +) +public class MarkUnknownLargestVariantLength extends StorageMigrationTool { + + @Override + protected void run() throws Exception { + for (String project : getVariantStorageProjects()) { + try (VariantStorageEngine engine = getVariantStorageEngineByProject(project)) { + for (Map.Entry entry : engine.getMetadataManager().getStudies().entrySet()) { + String studyName = entry.getKey(); + Integer studyId = entry.getValue(); + logger.info("Process study '" + studyName + "' (" + studyId + ")"); + // Check for indexed samples with undefined largest variant length + for (SampleMetadata sampleMetadata : engine.getMetadataManager().sampleMetadataIterable(studyId)) { + if (sampleMetadata.isIndexed()) { + if (!sampleMetadata.getAttributes().containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH)) { + logger.info("Mark unknown largest variant length for sample '" + sampleMetadata.getName() + "'"); + engine.getMetadataManager().updateSampleMetadata(studyId, sampleMetadata.getId(), + sm -> sm.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, true)); + } + } + } + } + } + } + } +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 1a074a17331..6437a0293b2 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -644,7 +644,6 @@ private void updateClinicalAnalysisIndividualReferences(ClientSession clientSess */ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document individual) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - String individualId = individual.getString(QueryParams.ID.key()); long individualUid = individual.getLong(PRIVATE_UID); long studyUid = individual.getLong(PRIVATE_STUDY_UID); @@ -653,8 +652,7 @@ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document .append(ClinicalAnalysisDBAdaptor.QueryParams.INDIVIDUAL.key(), individualUid); OpenCGAResult count = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().count(clientSession, query); if (count.getNumMatches() > 0) { - throw new CatalogDBException("Could not delete individual '" + individualId + "'. Individual is in use in " - + count.getNumMatches() + " cases"); + throw new CatalogDBException("Individual is in use in " + count.getNumMatches() + " cases"); } } @@ -940,7 +938,7 @@ public OpenCGAResult delete(Individual individual) throws CatalogDBException, Ca return runTransaction(clientSession -> privateDelete(clientSession, result.first())); } catch (CatalogDBException e) { logger.error("Could not delete individual {}: {}", individual.getId(), e.getMessage(), e); - throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e.getCause()); + throw new CatalogDBException("Could not delete individual " + individual.getId() + ": " + e.getMessage(), e); } } @@ -976,7 +974,7 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document indivi checkInUseInClinicalAnalysis(clientSession, individualDocument); logger.debug("Deleting individual {} ({})", individualId, individualUid); - // Look for all the different family versions + // Look for all the different individual versions Query individualQuery = new Query() .append(QueryParams.UID.key(), individualUid) .append(QueryParams.STUDY_UID.key(), studyUid); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index dde62bd87d3..6d7bd764c57 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -500,7 +500,6 @@ void updateIndividualIdFromSamples(ClientSession clientSession, long studyUid, S */ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document sample) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { - String sampleId = sample.getString(QueryParams.ID.key()); long sampleUid = sample.getLong(QueryParams.UID.key()); long studyUid = sample.getLong(QueryParams.STUDY_UID.key()); @@ -509,8 +508,7 @@ private void checkInUseInClinicalAnalysis(ClientSession clientSession, Document .append(ClinicalAnalysisDBAdaptor.QueryParams.SAMPLE.key(), sampleUid); OpenCGAResult count = dbAdaptorFactory.getClinicalAnalysisDBAdaptor().count(clientSession, query); if (count.getNumMatches() > 0) { - throw new CatalogDBException("Could not delete sample '" + sampleId + "'. Sample is in use in " - + count.getNumMatches() + " cases"); + throw new CatalogDBException("Sample is in use in " + count.getNumMatches() + " cases"); } } @@ -860,7 +858,7 @@ public OpenCGAResult delete(Sample sample) throws CatalogDBException, CatalogPar return runTransaction(clientSession -> privateDelete(clientSession, result.first())); } catch (CatalogDBException e) { logger.error("Could not delete sample {}: {}", sample.getId(), e.getMessage(), e); - throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e.getCause()); + throw new CatalogDBException("Could not delete sample " + sample.getId() + ": " + e.getMessage(), e); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java index d7183adabae..c135e390744 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java @@ -530,7 +530,8 @@ private void checkCanBeDeleted(Study study, Family family) throws CatalogExcepti OpenCGAResult result = clinicalDBAdaptor.get(query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); if (result.getNumResults() > 0) { String clinicalIds = result.getResults().stream().map(ClinicalAnalysis::getId).collect(Collectors.joining(", ")); - throw new CatalogException("Family {" + family.getId() + "} in use in Clinical Analyses: {" + clinicalIds + "}"); + throw new CatalogException("Could not delete family '" + family.getId() + "'. Family is in use in Clinical Analyses: '" + + clinicalIds + "'"); } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java index 07d03a295ce..97057677b0d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java @@ -710,10 +710,8 @@ private OpenCGAResult delete(Study study, Individual individual, ObjectMap param // Check if the individual can be deleted if (!params.getBoolean(Constants.FORCE, false)) { if (familyDataResult.getNumResults() > 0) { - throw new CatalogException("Individual found in the families: " + familyDataResult.getResults() - .stream() - .map(Family::getId) - .collect(Collectors.joining(", "))); + throw new CatalogException("Could not delete individual '" + individual.getId() + "'. Individual found in the families: " + + familyDataResult.getResults().stream().map(Family::getId).collect(Collectors.joining(", "))); } } else { logger.info("Forcing deletion of individuals belonging to families"); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java index 5ab592050d2..580bc1c2c2c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java @@ -815,6 +815,7 @@ public OpenCGAResult resetAnnotations(String studyStr, String sampleStr, } private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) throws CatalogException { + String msg = "Could not delete sample '" + sample.getId() + "'. "; // Look for files related with the sample Query query = new Query() .append(FileDBAdaptor.QueryParams.SAMPLE_IDS.key(), sample.getId()) @@ -834,9 +835,9 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) } if (!errorFiles.isEmpty()) { if (force) { - throw new CatalogException("Associated files are used in storage: " + StringUtils.join(errorFiles, ", ")); + throw new CatalogException(msg + "Associated files are used in storage: " + StringUtils.join(errorFiles, ", ")); } else { - throw new CatalogException("Sample associated to the files: " + StringUtils.join(errorFiles, ", ")); + throw new CatalogException(msg + "Sample associated to the files: " + StringUtils.join(errorFiles, ", ")); } } @@ -865,14 +866,14 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) } } if (associatedToDefaultCohort) { - throw new CatalogException("Sample in cohort " + StudyEntry.DEFAULT_COHORT); + throw new CatalogException(msg + "Sample in cohort " + StudyEntry.DEFAULT_COHORT); } if (!errorCohorts.isEmpty()) { if (force) { - throw new CatalogException("Sample present in cohorts in the process of calculating the stats: " + throw new CatalogException(msg + "Sample present in cohorts in the process of calculating the stats: " + StringUtils.join(errorCohorts, ", ")); } else { - throw new CatalogException("Sample present in cohorts: " + StringUtils.join(errorCohorts, ", ")); + throw new CatalogException(msg + "Sample present in cohorts: " + StringUtils.join(errorCohorts, ", ")); } } @@ -884,8 +885,7 @@ private void checkSampleCanBeDeleted(long studyId, Sample sample, boolean force) OpenCGAResult individualDataResult = individualDBAdaptor.get(query, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.UID.key(), IndividualDBAdaptor.QueryParams.ID.key()))); if (individualDataResult.getNumResults() > 0) { - throw new CatalogException("Sample from individual " + individualDataResult.first().getName() + "(" - + individualDataResult.first().getUid() + ")"); + throw new CatalogException(msg + "Sample is associated with individual '" + individualDataResult.first().getId() + "'."); } } } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java index 51c78472790..e9134475dd7 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/SampleManagerTest.java @@ -20,8 +20,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.RandomStringUtils; import org.bson.Document; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.internal.matchers.ThrowableMessageMatcher; import org.opencb.biodata.models.clinical.Disorder; import org.opencb.biodata.models.clinical.Phenotype; import org.opencb.biodata.models.clinical.qc.SampleQcVariantStats; @@ -2776,8 +2779,9 @@ public void deleteInUseInCATest() throws CatalogException { try { catalogManager.getSampleManager().delete(studyFqn, Collections.singletonList(case1.getProband().getSamples().get(0).getId()), new QueryOptions(ParamConstants.FORCE, true), token); + fail("Expected CatalogException with message containing 'in use in 3 cases'"); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("in use in 3 cases")); + MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("in use in 3 cases"))); } // unlock case3 @@ -2786,9 +2790,10 @@ public void deleteInUseInCATest() throws CatalogException { try { catalogManager.getSampleManager().delete(studyFqn, Collections.singletonList(case1.getProband().getSamples().get(0).getId()), - new QueryOptions(ParamConstants.FORCE, true), token); + new QueryOptions(ParamConstants.FORCE, false), token); + fail("Expected CatalogException with message containing 'associated with individual'."); } catch (CatalogException e) { - assertTrue(e.getMessage().contains("in use in 3 cases")); + MatcherAssert.assertThat(e, ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("associated with individual"))); } } diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index f36ae979df6..e2e650759c0 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Alignment-methods.R b/opencga-client/src/main/R/R/Alignment-methods.R index 7f1a6a2d745..517d51dad14 100644 --- a/opencga-client/src/main/R/R/Alignment-methods.R +++ b/opencga-client/src/main/R/R/Alignment-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index 5756f579584..8b679bba8af 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -1,6 +1,6 @@ # ############################################################################## ## UserClient -setGeneric("userClient", function(OpencgaR, filterId, user, users, endpointName, params=NULL, ...) +setGeneric("userClient", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) standardGeneric("userClient")) # ############################################################################## @@ -10,12 +10,12 @@ setGeneric("projectClient", function(OpencgaR, projects, project, endpointName, # ############################################################################## ## StudyClient -setGeneric("studyClient", function(OpencgaR, study, group, variableSet, studies, members, templateId, endpointName, params=NULL, ...) +setGeneric("studyClient", function(OpencgaR, members, templateId, study, variableSet, group, studies, endpointName, params=NULL, ...) standardGeneric("studyClient")) # ############################################################################## ## FileClient -setGeneric("fileClient", function(OpencgaR, file, files, folder, members, annotationSet, endpointName, params=NULL, ...) +setGeneric("fileClient", function(OpencgaR, members, folder, annotationSet, files, file, endpointName, params=NULL, ...) standardGeneric("fileClient")) # ############################################################################## @@ -25,22 +25,22 @@ setGeneric("jobClient", function(OpencgaR, members, job, jobs, endpointName, par # ############################################################################## ## SampleClient -setGeneric("sampleClient", function(OpencgaR, sample, members, annotationSet, samples, endpointName, params=NULL, ...) +setGeneric("sampleClient", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) standardGeneric("sampleClient")) # ############################################################################## ## IndividualClient -setGeneric("individualClient", function(OpencgaR, members, annotationSet, individuals, individual, endpointName, params=NULL, ...) +setGeneric("individualClient", function(OpencgaR, members, individuals, individual, annotationSet, endpointName, params=NULL, ...) standardGeneric("individualClient")) # ############################################################################## ## FamilyClient -setGeneric("familyClient", function(OpencgaR, families, members, annotationSet, family, endpointName, params=NULL, ...) +setGeneric("familyClient", function(OpencgaR, members, families, family, annotationSet, endpointName, params=NULL, ...) standardGeneric("familyClient")) # ############################################################################## ## CohortClient -setGeneric("cohortClient", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) +setGeneric("cohortClient", function(OpencgaR, members, cohorts, cohort, annotationSet, endpointName, params=NULL, ...) standardGeneric("cohortClient")) # ############################################################################## @@ -60,7 +60,7 @@ setGeneric("variantClient", function(OpencgaR, endpointName, params=NULL, ...) # ############################################################################## ## ClinicalClient -setGeneric("clinicalClient", function(OpencgaR, interpretation, interpretations, clinicalAnalysis, clinicalAnalyses, members, endpointName, params=NULL, ...) +setGeneric("clinicalClient", function(OpencgaR, interpretation, members, interpretations, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) standardGeneric("clinicalClient")) # ############################################################################## diff --git a/opencga-client/src/main/R/R/Clinical-methods.R b/opencga-client/src/main/R/R/Clinical-methods.R index 5d84fa14614..5d26c3f4c93 100644 --- a/opencga-client/src/main/R/R/Clinical-methods.R +++ b/opencga-client/src/main/R/R/Clinical-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -58,7 +58,7 @@ #' [*]: Required parameter #' @export -setMethod("clinicalClient", "OpencgaR", function(OpencgaR, interpretation, interpretations, clinicalAnalysis, clinicalAnalyses, members, endpointName, params=NULL, ...) { +setMethod("clinicalClient", "OpencgaR", function(OpencgaR, interpretation, members, interpretations, clinicalAnalyses, clinicalAnalysis, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/analysis/clinical/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Cohort-methods.R b/opencga-client/src/main/R/R/Cohort-methods.R index d192d6feec1..fb761746d4e 100644 --- a/opencga-client/src/main/R/R/Cohort-methods.R +++ b/opencga-client/src/main/R/R/Cohort-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("cohortClient", "OpencgaR", function(OpencgaR, members, annotationSet, cohort, cohorts, endpointName, params=NULL, ...) { +setMethod("cohortClient", "OpencgaR", function(OpencgaR, members, cohorts, cohort, annotationSet, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/cohorts/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Family-methods.R b/opencga-client/src/main/R/R/Family-methods.R index c582b23a93a..2b4ec1fe2e8 100644 --- a/opencga-client/src/main/R/R/Family-methods.R +++ b/opencga-client/src/main/R/R/Family-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("familyClient", "OpencgaR", function(OpencgaR, families, members, annotationSet, family, endpointName, params=NULL, ...) { +setMethod("familyClient", "OpencgaR", function(OpencgaR, members, families, family, annotationSet, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/families/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/File-methods.R b/opencga-client/src/main/R/R/File-methods.R index f3eeac80df2..75e2b732758 100644 --- a/opencga-client/src/main/R/R/File-methods.R +++ b/opencga-client/src/main/R/R/File-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ #' [*]: Required parameter #' @export -setMethod("fileClient", "OpencgaR", function(OpencgaR, file, files, folder, members, annotationSet, endpointName, params=NULL, ...) { +setMethod("fileClient", "OpencgaR", function(OpencgaR, members, folder, annotationSet, files, file, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/files/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/GA4GH-methods.R b/opencga-client/src/main/R/R/GA4GH-methods.R index 639683e171c..e0db9d9f3ae 100644 --- a/opencga-client/src/main/R/R/GA4GH-methods.R +++ b/opencga-client/src/main/R/R/GA4GH-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Individual-methods.R b/opencga-client/src/main/R/R/Individual-methods.R index 1839f2cb8c9..3e808df9e76 100644 --- a/opencga-client/src/main/R/R/Individual-methods.R +++ b/opencga-client/src/main/R/R/Individual-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("individualClient", "OpencgaR", function(OpencgaR, members, annotationSet, individuals, individual, endpointName, params=NULL, ...) { +setMethod("individualClient", "OpencgaR", function(OpencgaR, members, individuals, individual, annotationSet, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/individuals/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index ed70e8df6cf..f01e5f44139 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Meta-methods.R b/opencga-client/src/main/R/R/Meta-methods.R index c278a0c47e4..aa4f8f38aaf 100644 --- a/opencga-client/src/main/R/R/Meta-methods.R +++ b/opencga-client/src/main/R/R/Meta-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Operation-methods.R b/opencga-client/src/main/R/R/Operation-methods.R index 11621e2cf87..56206dd1df2 100644 --- a/opencga-client/src/main/R/R/Operation-methods.R +++ b/opencga-client/src/main/R/R/Operation-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Panel-methods.R b/opencga-client/src/main/R/R/Panel-methods.R index 20fb425cc31..45b9913a4b1 100644 --- a/opencga-client/src/main/R/R/Panel-methods.R +++ b/opencga-client/src/main/R/R/Panel-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Project-methods.R b/opencga-client/src/main/R/R/Project-methods.R index 80d88c4cc79..acea3c5c36d 100644 --- a/opencga-client/src/main/R/R/Project-methods.R +++ b/opencga-client/src/main/R/R/Project-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/R/R/Sample-methods.R b/opencga-client/src/main/R/R/Sample-methods.R index 32e64a3ecc9..d9f94ce2011 100644 --- a/opencga-client/src/main/R/R/Sample-methods.R +++ b/opencga-client/src/main/R/R/Sample-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -39,7 +39,7 @@ #' [*]: Required parameter #' @export -setMethod("sampleClient", "OpencgaR", function(OpencgaR, sample, members, annotationSet, samples, endpointName, params=NULL, ...) { +setMethod("sampleClient", "OpencgaR", function(OpencgaR, members, sample, annotationSet, samples, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/samples/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index a9c9a3c90f8..9659a1d59e3 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ #' [*]: Required parameter #' @export -setMethod("studyClient", "OpencgaR", function(OpencgaR, study, group, variableSet, studies, members, templateId, endpointName, params=NULL, ...) { +setMethod("studyClient", "OpencgaR", function(OpencgaR, members, templateId, study, variableSet, group, studies, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/studies/acl/{members}/update: diff --git a/opencga-client/src/main/R/R/User-methods.R b/opencga-client/src/main/R/R/User-methods.R index 2a1d1bc3e82..91cba91802d 100644 --- a/opencga-client/src/main/R/R/User-methods.R +++ b/opencga-client/src/main/R/R/User-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. @@ -38,7 +38,7 @@ #' [*]: Required parameter #' @export -setMethod("userClient", "OpencgaR", function(OpencgaR, filterId, user, users, endpointName, params=NULL, ...) { +setMethod("userClient", "OpencgaR", function(OpencgaR, user, filterId, users, endpointName, params=NULL, ...) { switch(endpointName, #' @section Endpoint /{apiVersion}/users/login: diff --git a/opencga-client/src/main/R/R/Variant-methods.R b/opencga-client/src/main/R/R/Variant-methods.R index ad760abcd16..3ad2940ffc6 100644 --- a/opencga-client/src/main/R/R/Variant-methods.R +++ b/opencga-client/src/main/R/R/Variant-methods.R @@ -2,7 +2,7 @@ # WARNING: AUTOGENERATED CODE # # This code was generated by a tool. -# Autogenerated on: 2023-09-12 +# Autogenerated on: 2023-10-20 # # Manual changes to this file may cause unexpected behavior in your application. # Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java index 58d37b4caf6..007c5f7f2e5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/AbstractParentClient.java @@ -456,6 +456,7 @@ protected RestResponse callUploadRest(WebTarget path, Map client.property(ClientProperties.READ_TIMEOUT, DEFAULT_UPLOAD_TIMEOUT); path.register(MultiPartFeature.class); path.property(ClientProperties.REQUEST_ENTITY_PROCESSING, RequestEntityProcessing.CHUNKED); + final FileDataBodyPart filePart = new FileDataBodyPart("file", new File(filePath)); FormDataMultiPart formDataMultiPart = new FormDataMultiPart(); // Add the rest of the parameters to the form diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java index 6aad0bc7842..4b1ffc655ed 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AdminClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Admin webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: admin */ public class AdminClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java index 16c679f642f..3f34d8a1ae5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/AlignmentClient.java @@ -40,7 +40,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -49,7 +49,7 @@ /** * This class contains methods for the Alignment webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: analysis/alignment */ public class AlignmentClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java index 94ab79f677d..8af49e36b9c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ClinicalAnalysisClient.java @@ -51,7 +51,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -60,7 +60,7 @@ /** * This class contains methods for the ClinicalAnalysis webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: analysis/clinical */ public class ClinicalAnalysisClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java index 2320ca7bbcd..79b1257e020 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/CohortClient.java @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ /** * This class contains methods for the Cohort webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: cohorts */ public class CohortClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java index ad9ef470eaa..fe5f78505a9 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/DiseasePanelClient.java @@ -35,7 +35,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -44,7 +44,7 @@ /** * This class contains methods for the DiseasePanel webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: panels */ public class DiseasePanelClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java index 94b0e7cf66d..abc8d908670 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FamilyClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Family webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: families */ public class FamilyClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java index 94eeead7197..10e7daa866f 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/FileClient.java @@ -43,7 +43,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -52,7 +52,7 @@ /** * This class contains methods for the File webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: files */ public class FileClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java index 48b4524ea7f..364a0f3fd71 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/GA4GHClient.java @@ -27,7 +27,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -36,7 +36,7 @@ /** * This class contains methods for the GA4GH webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: ga4gh */ public class GA4GHClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java index b89c47024df..0dc532fa526 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/IndividualClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Individual webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: individuals */ public class IndividualClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index 7945a8d65cc..846bc56452e 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -37,7 +37,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -46,7 +46,7 @@ /** * This class contains methods for the Job webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: jobs */ public class JobClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java index 794c6ad3394..46feb9030f8 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/MetaClient.java @@ -28,7 +28,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -37,7 +37,7 @@ /** * This class contains methods for the Meta webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: meta */ public class MetaClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java index 317a4af226c..f4dd61aaaf0 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ProjectClient.java @@ -32,7 +32,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -41,7 +41,7 @@ /** * This class contains methods for the Project webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: projects */ public class ProjectClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java index 5cbb8f10fd4..5e42759715a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/SampleClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the Sample webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: samples */ public class SampleClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java index 8f018861260..5c8e425988a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/StudyClient.java @@ -45,7 +45,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -54,7 +54,7 @@ /** * This class contains methods for the Study webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: studies */ public class StudyClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java index adb4ce85938..8f340c1e1c0 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserClient.java @@ -36,7 +36,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -45,7 +45,7 @@ /** * This class contains methods for the User webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: users */ public class UserClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java index 4801650a0f9..f89aca9752a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantClient.java @@ -62,7 +62,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -71,7 +71,7 @@ /** * This class contains methods for the Variant webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: analysis/variant */ public class VariantClient extends AbstractParentClient { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java index c1ffc952c9d..a6ad3261f56 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/VariantOperationClient.java @@ -50,7 +50,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. -* Autogenerated on: 2023-09-12 +* Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. @@ -59,7 +59,7 @@ /** * This class contains methods for the VariantOperation webservices. - * Client version: 2.11.0-SNAPSHOT + * Client version: 2.12.0-SNAPSHOT * PATH: operation */ public class VariantOperationClient extends AbstractParentClient { diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index 2466edfccfd..d3430779824 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Alignment.js b/opencga-client/src/main/javascript/Alignment.js index 446219c5a59..7fed5e60f52 100644 --- a/opencga-client/src/main/javascript/Alignment.js +++ b/opencga-client/src/main/javascript/Alignment.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/ClinicalAnalysis.js b/opencga-client/src/main/javascript/ClinicalAnalysis.js index 221fa4a68da..871a204c82b 100644 --- a/opencga-client/src/main/javascript/ClinicalAnalysis.js +++ b/opencga-client/src/main/javascript/ClinicalAnalysis.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Cohort.js b/opencga-client/src/main/javascript/Cohort.js index 9d070dc8ec8..50dc8bd3d78 100644 --- a/opencga-client/src/main/javascript/Cohort.js +++ b/opencga-client/src/main/javascript/Cohort.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/DiseasePanel.js b/opencga-client/src/main/javascript/DiseasePanel.js index 1d97f0d3b6f..39643fabe88 100644 --- a/opencga-client/src/main/javascript/DiseasePanel.js +++ b/opencga-client/src/main/javascript/DiseasePanel.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Family.js b/opencga-client/src/main/javascript/Family.js index 5b11ceacd3a..e69b16398cd 100644 --- a/opencga-client/src/main/javascript/Family.js +++ b/opencga-client/src/main/javascript/Family.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/File.js b/opencga-client/src/main/javascript/File.js index 27d76b733e7..f91c4393880 100644 --- a/opencga-client/src/main/javascript/File.js +++ b/opencga-client/src/main/javascript/File.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/GA4GH.js b/opencga-client/src/main/javascript/GA4GH.js index 2e3f968ab05..e7a55972061 100644 --- a/opencga-client/src/main/javascript/GA4GH.js +++ b/opencga-client/src/main/javascript/GA4GH.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Individual.js b/opencga-client/src/main/javascript/Individual.js index 160bb589c50..f4df47e4855 100644 --- a/opencga-client/src/main/javascript/Individual.js +++ b/opencga-client/src/main/javascript/Individual.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index 52f5cdde599..e07556da3d5 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Meta.js b/opencga-client/src/main/javascript/Meta.js index 0207e95847a..d9b2cd2e881 100644 --- a/opencga-client/src/main/javascript/Meta.js +++ b/opencga-client/src/main/javascript/Meta.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Project.js b/opencga-client/src/main/javascript/Project.js index f43c7394d71..9b1cf62bd1d 100644 --- a/opencga-client/src/main/javascript/Project.js +++ b/opencga-client/src/main/javascript/Project.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Sample.js b/opencga-client/src/main/javascript/Sample.js index edefccf3e5a..3839ed68aab 100644 --- a/opencga-client/src/main/javascript/Sample.js +++ b/opencga-client/src/main/javascript/Sample.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index d153ba9d316..67950c510c5 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/User.js b/opencga-client/src/main/javascript/User.js index d8c60b8d409..b4a22fd6ca6 100644 --- a/opencga-client/src/main/javascript/User.js +++ b/opencga-client/src/main/javascript/User.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/Variant.js b/opencga-client/src/main/javascript/Variant.js index 35617fde07f..cdf8033409b 100644 --- a/opencga-client/src/main/javascript/Variant.js +++ b/opencga-client/src/main/javascript/Variant.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/javascript/VariantOperation.js b/opencga-client/src/main/javascript/VariantOperation.js index daa6917728a..2bd14691589 100644 --- a/opencga-client/src/main/javascript/VariantOperation.js +++ b/opencga-client/src/main/javascript/VariantOperation.js @@ -12,7 +12,7 @@ * WARNING: AUTOGENERATED CODE * * This code was generated by a tool. - * Autogenerated on: 2023-09-12 + * Autogenerated on: 2023-10-20 * * Manual changes to this file may cause unexpected behavior in your application. * Manual changes to this file will be overwritten if the code is regenerated. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index 28598b602eb..9d3c5c70e4b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Admin(_ParentRestClient): """ This class contains methods for the 'Admin' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/admin """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py index d19895d21ba..ee6e3dec87e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/alignment_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Alignment(_ParentRestClient): """ This class contains methods for the 'Analysis - Alignment' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/analysis/alignment """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py index cf8de0798c5..dd7e8445d6d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/clinical_analysis_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class ClinicalAnalysis(_ParentRestClient): """ This class contains methods for the 'Analysis - Clinical' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/analysis/clinical """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py index f7d6ad80542..00b36a955c4 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/cohort_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Cohort(_ParentRestClient): """ This class contains methods for the 'Cohorts' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/cohorts """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py index 6d47ea09d3c..32c3fcad43a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/disease_panel_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class DiseasePanel(_ParentRestClient): """ This class contains methods for the 'Disease Panels' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/panels """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py index 65e3a070dce..cca2f512f1e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/family_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Family(_ParentRestClient): """ This class contains methods for the 'Families' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/families """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py index c6c028d3b3a..56440b2919c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/file_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class File(_ParentRestClient): """ This class contains methods for the 'Files' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/files """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py index 48e5b17de75..1a5348f7c6d 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/ga4gh_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class GA4GH(_ParentRestClient): """ This class contains methods for the 'GA4GH' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/ga4gh """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py index 07dd3d04b0d..cd2763c5443 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/individual_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Individual(_ParentRestClient): """ This class contains methods for the 'Individuals' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/individuals """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index d692f8e099c..45b947e2098 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Job(_ParentRestClient): """ This class contains methods for the 'Jobs' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/jobs """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py index d43227b33f8..6abaf5e1f17 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/meta_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Meta(_ParentRestClient): """ This class contains methods for the 'Meta' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/meta """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py index bd29b4b1013..c6c1b44ae5a 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/project_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Project(_ParentRestClient): """ This class contains methods for the 'Projects' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/projects """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py index fcb5c944884..8a257daeadd 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/sample_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Sample(_ParentRestClient): """ This class contains methods for the 'Samples' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/samples """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 40cc9a63c08..0b900697dab 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Study(_ParentRestClient): """ This class contains methods for the 'Studies' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/studies """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py index 3f2ed4eae3e..5b920d96a6e 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class User(_ParentRestClient): """ This class contains methods for the 'Users' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/users """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py index 80e1746470a..be6b137a26f 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class Variant(_ParentRestClient): """ This class contains methods for the 'Analysis - Variant' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/analysis/variant """ diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py index 4297fbf08ae..d096a751c9b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/variant_operation_client.py @@ -2,7 +2,7 @@ WARNING: AUTOGENERATED CODE This code was generated by a tool. - Autogenerated on: 2023-09-12 + Autogenerated on: 2023-10-20 Manual changes to this file may cause unexpected behavior in your application. Manual changes to this file will be overwritten if the code is regenerated. @@ -14,7 +14,7 @@ class VariantOperation(_ParentRestClient): """ This class contains methods for the 'Operations - Variant Storage' webservices - Client version: 2.11.0-SNAPSHOT + Client version: 2.12.0-SNAPSHOT PATH: /{apiVersion}/operation """ diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java index edf5e3e50bc..6d7eafd1503 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java @@ -11,17 +11,20 @@ import org.opencb.cellbase.core.result.CellBaseDataResponse; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.utils.VersionUtils; + import org.opencb.opencga.core.config.storage.CellBaseConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; + import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; + public class CellBaseValidator { private static Logger logger = LoggerFactory.getLogger(CellBaseValidator.class); @@ -193,6 +196,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException } } } + String apiKey = getApiKey(); if (StringUtils.isEmpty(apiKey)) { cellBaseConfiguration.setApiKey(null); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java index aea655f6d9b..1e0cd7dac49 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/variant/VariantIndexParams.java @@ -17,6 +17,8 @@ package org.opencb.opencga.core.models.variant; import org.opencb.biodata.models.variant.metadata.Aggregation; +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.tools.ToolParams; public class VariantIndexParams extends ToolParams { @@ -29,11 +31,14 @@ public VariantIndexParams(String file, boolean resume, String outdir, boolean transform, boolean gvcf, - boolean normalizationSkip, String referenceGenome, + boolean normalizationSkip, + String referenceGenome, String failOnMalformedLines, boolean family, boolean somatic, - boolean load, String loadSplitData, boolean loadMultiFileData, + boolean load, + boolean forceReload, + String loadSplitData, boolean loadMultiFileData, String loadSampleIndex, String loadArchive, String loadHomRef, @@ -53,6 +58,7 @@ public VariantIndexParams(String file, this.family = family; this.somatic = somatic; this.load = load; + this.forceReload = forceReload; this.loadSplitData = loadSplitData; this.loadMultiFileData = loadMultiFileData; this.loadSampleIndex = loadSampleIndex; @@ -73,43 +79,82 @@ public VariantIndexParams(String file, this.skipIndexedFiles = skipIndexedFiles; } + @DataField(description = "List of files to be indexed.") private String file; + @DataField(description = "Resume a previously failed index operation") private boolean resume; + @DataField(description = "Output directory") private String outdir; + @DataField(description = "If present it only runs the transform stage, no load is executed") private boolean transform; + @DataField(description = "Hint to indicate that the input file is in gVCF format.") private boolean gvcf; + @DataField(description = "Do not execute the normalization process. WARN: INDELs will be stored with the context base") private boolean normalizationSkip; + @DataField(description = "Reference genome in FASTA format used during the normalization step " + + "for a complete left alignment") private String referenceGenome; + @DataField(description = "Fail when encountering malformed lines. (yes, no, auto) [auto]") private String failOnMalformedLines; + @DataField(description = "Indicate that the files to be loaded are part of a family. " + + "This will set 'load-hom-ref' to YES if it was in AUTO and execute 'family-index' afterwards") private boolean family; + @DataField(description = "Indicate that the files to be loaded contain somatic samples. " + + "This will set 'load-hom-ref' to YES if it was in AUTO.") private boolean somatic; + @DataField(description = "If present only the load stage is executed, transformation is skipped") private boolean load; + @DataField(description = "If the file is already loaded, force a file reload") + private boolean forceReload; + @DataField(description = "Indicate that the variants from a group of samples is split in multiple files, either by CHROMOSOME or by REGION. In either case, variants from different files must not overlap.") private String loadSplitData; + @DataField(description = "Indicate the presence of multiple files for the same sample. Each file could be the result of a different vcf-caller or experiment over the same sample.") private boolean loadMultiFileData; + @DataField(description = "Build sample index while loading. (yes, no, auto) [auto]") private String loadSampleIndex; + @DataField(description = "Load archive data. (yes, no, auto) [auto]") private String loadArchive; + @DataField(description = "Load HOM_REF genotypes. (yes, no, auto) [auto]") private String loadHomRef; + @DataField(description = "Execute post load checks over the database. (yes, no, auto) [auto]") private String postLoadCheck; + @DataField(description = "Load the genotype data for the current file. " + + "This only applies to the GT field from the FORMAT. All the rest of fields from the INFO and FORMAT will be loaded. " + + "Use this parameter skip load data when the GT field is not reliable, or its only value across the file is \"./.\". " + + "If \"auto\", genotypes will be automatically excluded if all genotypes are either missing, ./. or 0/0. " + + "(yes, no, auto) [auto]") private String includeGenotypes; + @DataField(description = "Index including other sample data fields (i.e. FORMAT fields)." + + " Use \"" + ParamConstants.ALL + "\", \"" + ParamConstants.NONE + "\", or CSV with the fields to load.") private String includeSampleData; + @DataField(deprecated = true, description = "Currently two levels of merge are supported: \"basic\" mode merge genotypes of the same variants while \"advanced\" merge multiallelic and overlapping variants.") private String merge; + @DataField(description = "Specify how duplicated variants should be handled. Available policies: \"discard\", \"maxQual\"") private String deduplicationPolicy; + @DataField(description = "Calculate indexed variants statistics after the load step") private boolean calculateStats; + @DataField(description = "Select the type of aggregated VCF file: none, basic, EVS or ExAC") private Aggregation aggregated; + @DataField(description = "File containing population names mapping in an aggregated VCF file") private String aggregationMappingFile; + @DataField(description = "Annotate indexed variants after the load step") private boolean annotate; + @DataField(description = "Annotation source {cellbase_rest, cellbase_db_adaptor}") private String annotator; + @DataField(description = "Overwrite annotations in variants already present") private boolean overwriteAnnotations; + @DataField(description = "Add files to the secondary search index") private boolean indexSearch; + @DataField(description = "Do not fail if any of the input files was already indexed.") private boolean skipIndexedFiles; public String getFile() { @@ -211,6 +256,15 @@ public VariantIndexParams setLoad(boolean load) { return this; } + public boolean isForceReload() { + return forceReload; + } + + public VariantIndexParams setForceReload(boolean forceReload) { + this.forceReload = forceReload; + return this; + } + public String getLoadSplitData() { return loadSplitData; } diff --git a/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java b/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java index ada81158f44..cab404b3c34 100644 --- a/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java +++ b/opencga-core/src/test/java/org/opencb/opencga/core/cellbase/CellBaseValidatorTest.java @@ -112,4 +112,5 @@ public void testUnsignedApiKey() throws IOException { CellBaseConfiguration validated = CellBaseValidator.validate(new CellBaseConfiguration(ParamConstants.CELLBASE_URL, "v5.4", null, apiKey), "hsapiens", "grch38", true); Assert.assertNotNull(validated.getApiKey()); } + } \ No newline at end of file diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/OpenCGAHealthCheckMonitor.java b/opencga-server/src/main/java/org/opencb/opencga/server/OpenCGAHealthCheckMonitor.java new file mode 100644 index 00000000000..fa8c0a1b95f --- /dev/null +++ b/opencga-server/src/main/java/org/opencb/opencga/server/OpenCGAHealthCheckMonitor.java @@ -0,0 +1,283 @@ +package org.opencb.opencga.server; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang3.time.StopWatch; +import org.opencb.commons.datastore.core.Event; +import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.config.Configuration; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.storage.core.StorageEngineFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +public class OpenCGAHealthCheckMonitor { + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final AtomicReference cachedResult = new AtomicReference<>(); + + private final Configuration configuration; + private final CatalogManager catalogManager; + private final StorageEngineFactory storageEngineFactory; + private final VariantStorageManager variantManager; + private final ExecutorService executorService; + + public OpenCGAHealthCheckMonitor(Configuration configuration, CatalogManager catalogManager, + StorageEngineFactory storageEngineFactory, + VariantStorageManager variantManager) { + this.configuration = configuration; + this.catalogManager = catalogManager; + this.storageEngineFactory = storageEngineFactory; + this.variantManager = variantManager; + executorService = Executors.newCachedThreadPool(); + } + + public static class HealthCheckStatus { + + enum Status { + OK, + KO, + NA + } + + @JsonProperty("CatalogMongoDB") + private Status catalogMongoDbStatus = null; + @JsonProperty("Solr") + private Status solrStatus = null; + @JsonProperty("VariantStorage") + private Status variantStorageStatus = null; + + @JsonProperty("VariantStorageId") + private String variantStorageId = ""; + + @JsonIgnore + private String errorMessage = null; + @JsonIgnore + private LocalDateTime creationDate; + @JsonIgnore + private boolean healthy; + + public HealthCheckStatus() { + } + + public Status getCatalogMongoDbStatus() { + return catalogMongoDbStatus; + } + + public HealthCheckStatus setCatalogMongoDbStatus(Status catalogMongoDbStatus) { + this.catalogMongoDbStatus = catalogMongoDbStatus; + return this; + } + + public Status getSolrStatus() { + return solrStatus; + } + + public HealthCheckStatus setSolrStatus(Status solrStatus) { + this.solrStatus = solrStatus; + return this; + } + + public Status getVariantStorageStatus() { + return variantStorageStatus; + } + + public HealthCheckStatus setVariantStorageStatus(Status variantStorageStatus) { + this.variantStorageStatus = variantStorageStatus; + return this; + } + + public String getVariantStorageId() { + return variantStorageId; + } + + public HealthCheckStatus setVariantStorageId(String variantStorageId) { + this.variantStorageId = variantStorageId; + return this; + } + + public boolean isHealthy() { + return healthy; + } + + public HealthCheckStatus setHealthy(boolean healthy) { + this.healthy = healthy; + return this; + } + + public String getErrorMessage() { + return errorMessage; + } + + public HealthCheckStatus setErrorMessage(String errorMessage) { + this.errorMessage = errorMessage; + return this; + } + + public LocalDateTime getCreationDate() { + return creationDate; + } + + public HealthCheckStatus setCreationDate(LocalDateTime creationDate) { + this.creationDate = creationDate; + return this; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("HealthCheckStatus{"); + sb.append("catalogMongoDbStatus='").append(catalogMongoDbStatus).append('\''); + sb.append(", solrStatus='").append(solrStatus).append('\''); + sb.append(", variantStorageStatus='").append(variantStorageStatus).append('\''); + sb.append(", variantStorageId='").append(variantStorageId).append('\''); + sb.append(", errorMessage='").append(errorMessage).append('\''); + sb.append(", creationDate=").append(creationDate); + sb.append(", healthy=").append(healthy); + sb.append('}'); + return sb.toString(); + } + } + + public void asyncUpdate() { + if (shouldUpdateStatus()) { + executorService.submit(() -> { + logger.debug("Update HealthCheck cache status"); + updateHealthCheck(); + }); + } + } + + public OpenCGAResult getStatus() { + + OpenCGAResult queryResult = new OpenCGAResult<>(); + StopWatch stopWatch = StopWatch.createStarted(); + + if (shouldUpdateStatus()) { + logger.debug("Update HealthCheck cache status"); + updateHealthCheck(); + } else { + HealthCheckStatus status = cachedResult.get(); + String msg = "HealthCheck results from cache at " + status.getCreationDate().format(DateTimeFormatter.ofPattern("HH:mm:ss")); + queryResult.setEvents(Collections.singletonList(new Event(Event.Type.INFO, msg))); + logger.debug(msg); + } + + queryResult.setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); + queryResult.setResults(Collections.singletonList(cachedResult.get())); + return queryResult; + } + + private boolean shouldUpdateStatus() { + HealthCheckStatus status = cachedResult.get(); + if (status == null || !status.isHealthy()) { + // Always update if not healthy or undefined + return true; + } + // If healthy, only update every "healthCheck.interval" seconds + long elapsedTime = Duration.between(status.getCreationDate(), LocalDateTime.now()).getSeconds(); + return elapsedTime > configuration.getHealthCheck().getInterval(); + } + + private void updateHealthCheck() { + updateHealthCheck(StopWatch.createStarted()); + } + + private synchronized void updateHealthCheck(StopWatch totalTime) { + if (!shouldUpdateStatus()) { + if (totalTime.getTime(TimeUnit.SECONDS) > 5) { + logger.warn("Slow OpenCGA status. Synchronized time wait: {} . Skip update.", + totalTime.getTime(TimeUnit.MILLISECONDS) / 1000.0 + ); + } + // Skip update! + return; + } + StringBuilder errorMsg = new StringBuilder(); + boolean healthy = true; + + HealthCheckStatus status = new HealthCheckStatus(); + + StopWatch catalogMongoDBTime = StopWatch.createStarted(); + try { + if (catalogManager.getCatalogDatabaseStatus()) { + status.setCatalogMongoDbStatus(HealthCheckStatus.Status.OK); + } else { + status.setCatalogMongoDbStatus(HealthCheckStatus.Status.KO); + healthy = false; + } + } catch (Exception e) { + status.setCatalogMongoDbStatus(HealthCheckStatus.Status.KO); + healthy = false; + errorMsg.append(e.getMessage()); + logger.error("Error reading catalog status", e); + } + catalogMongoDBTime.stop(); + + StopWatch storageTime = StopWatch.createStarted(); + try { + storageEngineFactory.getVariantStorageEngine(null, configuration.getDatabasePrefix() + "_test_connection", "test_connection") + .testConnection(); + status.setVariantStorageId(storageEngineFactory.getVariantStorageEngine().getStorageEngineId()); + status.setVariantStorageStatus(HealthCheckStatus.Status.OK); + } catch (Exception e) { + status.setVariantStorageStatus(HealthCheckStatus.Status.KO); + healthy = false; + errorMsg.append(e.getMessage()); + logger.error("Error reading variant storage status", e); + } + storageTime.stop(); + + StopWatch solrEngineTime = StopWatch.createStarted(); + if (storageEngineFactory.getStorageConfiguration().getSearch().isActive()) { + try { + if (variantManager.isSolrAvailable()) { + status.setSolrStatus(HealthCheckStatus.Status.OK); + } else { + errorMsg.append(", unable to connect with solr, "); + status.setSolrStatus(HealthCheckStatus.Status.KO); + healthy = false; + } + } catch (Exception e) { + status.setSolrStatus(HealthCheckStatus.Status.KO); + healthy = false; + errorMsg.append(e.getMessage()); + logger.error("Error reading solr status", e); + } + } else { + status.setSolrStatus(HealthCheckStatus.Status.NA); + } + solrEngineTime.stop(); + totalTime.stop(); + + if (totalTime.getTime(TimeUnit.SECONDS) > 5) { + logger.warn("Slow OpenCGA status: Updated time: {}. Catalog: {} , Storage: {} , Solr: {}", + totalTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, + catalogMongoDBTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, + storageTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, + solrEngineTime.getTime(TimeUnit.MILLISECONDS) / 1000.0 + ); + } + + if (errorMsg.length() == 0) { + status.setErrorMessage(null); + } else { + status.setErrorMessage(errorMsg.toString()); + } + + status.setCreationDate(LocalDateTime.now()); + status.setHealthy(healthy); + cachedResult.set(status); + } + +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java index d814ac76f1b..74874049837 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/ClientsGeneratorMain.java @@ -70,6 +70,7 @@ public static void main(String[] args) throws URISyntaxException { clientsGenerator.cli(flatRestApi); } catch (Exception e) { logger.error(e.getMessage(), e); + System.exit(1); } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java index 96b13ca4292..ea7f6fe917b 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/CategoryConfig.java @@ -1,6 +1,8 @@ package org.opencb.opencga.server.generator.config; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; public class CategoryConfig { @@ -186,6 +188,21 @@ public boolean isAvailableCommand(String commandName) { return true; } + public Command getCommandOrFail(String commandName) { + Command command = getCommand(commandName); + if (command == null) { + List availableCommands; + if (commands == null) { + availableCommands = Collections.emptyList(); + } else { + availableCommands = commands.stream().map(Command::getName).collect(Collectors.toList()); + } + throw new IllegalArgumentException("Command '" + commandName + "' not found. Given category '" + name + "'" + + ", available sub-commands are: " + availableCommands + "."); + } + return command; + } + public Command getCommand(String commandName) { if (commands != null) { for (Command cmd : commands) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java index 72857bc2aec..86f1bcda615 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.server.generator.config.CategoryConfig; +import org.opencb.opencga.server.generator.config.Command; import org.opencb.opencga.server.generator.config.CommandLineConfiguration; import org.opencb.opencga.server.generator.models.RestApi; import org.opencb.opencga.server.generator.models.RestCategory; @@ -186,8 +187,6 @@ public static String getCommandName(RestCategory restCategory, RestEndpoint rest } protected static String getMethodName(RestCategory restCategory, RestEndpoint restEndpoint) { - - String methodName = ""; String subpath = restEndpoint.getPath().replace(restCategory.getPath() + "/", ""); return getMethodName(subpath); } @@ -298,4 +297,61 @@ public boolean isValidMap(RestParameter bodyRestParameter) { || "java.util.Map".equals(bodyRestParameter.getGenericType()) || "java.util.Map".equals(bodyRestParameter.getGenericType()); } + + protected String getExecutorVarName(RestCategory restCategory) { + return "custom" + getAsClassName(restCategory.getName()) + "CommandExecutor"; + } + + protected String getExecutorClassName(RestCategory restCategory, CategoryConfig categoryConfig, String commandName) { + String executorClassName; + Command command = categoryConfig.getCommandOrFail(commandName); + if (StringUtils.isNotEmpty(command.getExecutorExtendedClassName())) { + executorClassName = command.getExecutorExtendedClassName(); + } else { + executorClassName = "Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor"; + } + return executorClassName; + } + + protected String getCommandOptionsClassName(RestCategory restCategory, CategoryConfig categoryConfig, String commandName) { + String commandOptionsClassName; + if (categoryConfig.isExtendedOptionCommand(commandName)) { + commandOptionsClassName = "Custom" + getAsClassName(restCategory.getName()) + "CommandOptions"; + } else { + commandOptionsClassName = getAsClassName(restCategory.getName()) + "CommandOptions"; + } + return commandOptionsClassName; + } + + protected String getCommandOptionsClassName(RestCategory restCategory) { + return getAsClassName(restCategory.getName()) + "CommandOptions"; + } + + protected String getSubCommandOptionsClassName(CategoryConfig categoryConfig, String commandName) { +// String methodName = getJavaMethodName(categoryConfig, commandName); + String methodName = getAsCamelCase(commandName); + String subCommandOptionsClassName = getAsClassName(methodName + "CommandOptions"); + return subCommandOptionsClassName; + } + + protected String getCommandOptionsVarName(RestCategory restCategory) { + return getAsVariableName(getAsCamelCase(restCategory.getName())) + "CommandOptions"; + } + + protected String getSubCommandOptionsVarName(CategoryConfig categoryConfig, String commandName) { +// String methodName = getJavaMethodName(categoryConfig, commandName); + String methodName = getAsCamelCase(commandName); + return methodName + "CommandOptions"; + } + + protected String getJavaMethodName(CategoryConfig config, String commandName) { + Command command = config.getCommand(commandName); + String commandMethod; + if (command != null && StringUtils.isNotEmpty(command.getRename())) { + commandMethod = command.getRename(); + } else { + commandMethod = getAsCamelCase(commandName); + } + return commandMethod; + } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java index f75776a7dfa..d0c830abd3d 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ExecutorsCliRestApiWriter.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.text.SimpleDateFormat; import java.util.*; public class ExecutorsCliRestApiWriter extends ParentClientRestApiWriter { @@ -44,48 +43,48 @@ public ExecutorsCliRestApiWriter(RestApi restApi, CommandLineConfiguration confi @Override protected String getClassImports(String key) { - StringBuilder sb = new StringBuilder(); RestCategory restCategory = availableCategories.get(key); CategoryConfig categoryConfig = availableCategoryConfigs.get(key); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - sb.append("package ").append(config.getOptions().getExecutorsPackage()).append(";\n\n"); - sb.append("import com.fasterxml.jackson.databind.DeserializationFeature;\n"); +// SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Set imports = new TreeSet<>(); + imports.add("com.fasterxml.jackson.databind.DeserializationFeature"); if (StringUtils.isEmpty(config.getApiConfig().getExecutorsParentClass())) { - sb.append("import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor;\n"); + imports.add("org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor"); } else { - sb.append("import " + config.getApiConfig().getExecutorsParentClass() + ";\n"); + imports.add(config.getApiConfig().getExecutorsParentClass()); } - sb.append("import org.opencb.opencga.app.cli.main.*;\n"); - sb.append("import org.opencb.opencga.core.response.RestResponse;\n"); - sb.append("import org.opencb.opencga.client.exceptions.ClientException;\n"); - sb.append("import org.opencb.commons.datastore.core.ObjectMap;\n\n"); - sb.append("import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException;\n"); - sb.append("import org.opencb.opencga.core.common.JacksonUtils;\n\n"); - sb.append("import com.fasterxml.jackson.databind.ObjectMapper;\n"); - sb.append("import java.util.List;\n"); - sb.append("import java.util.HashMap;\n"); - sb.append("import org.opencb.opencga.core.response.QueryType;\n"); - sb.append("import org.opencb.commons.utils.PrintUtils;\n\n"); + imports.add("com.fasterxml.jackson.databind.DeserializationFeature"); + imports.add("org.opencb.opencga.app.cli.main.*"); + imports.add("org.opencb.opencga.core.response.RestResponse"); + imports.add("org.opencb.opencga.client.exceptions.ClientException"); + imports.add("org.opencb.commons.datastore.core.ObjectMap"); + imports.add("org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException"); + imports.add("org.opencb.opencga.core.common.JacksonUtils"); + imports.add("com.fasterxml.jackson.databind.ObjectMapper"); + imports.add("java.util.List"); + imports.add("java.util.HashMap"); + imports.add("org.opencb.opencga.core.response.QueryType"); + imports.add("org.opencb.commons.utils.PrintUtils"); + // Add custom parent class - sb.append("import " + config.getOptions().getOptionsPackage() + "." + getAsClassName(restCategory.getName()) + "CommandOptions;\n\n"); + imports.add(config.getOptions().getOptionsPackage() + "." + getCommandOptionsClassName(restCategory)); if (categoryConfig.isExecutorExtended()) { for (RestEndpoint restEndpoint : restCategory.getEndpoints()) { String commandName = getCommandName(restCategory, restEndpoint); - if (categoryConfig.getCommand(commandName) != null && StringUtils.isNotEmpty(categoryConfig.getCommand(commandName).getExecutorExtendedClassName())) { - sb.append("import " + categoryConfig.getCommand(commandName).getExecutorExtendedClassName() + ";\n"); + Command command = categoryConfig.getCommand(commandName); + if (command != null && StringUtils.isNotEmpty(command.getExecutorExtendedClassName())) { + imports.add(command.getExecutorExtendedClassName()); } else { - sb.append("import org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor;\n"); + imports.add("org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor"); } if (categoryConfig.isExtendedOptionCommand(commandName)) { - sb.append("import org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandOptions;\n"); - + imports.add("org.opencb.opencga.app.cli.main.custom.Custom" + getAsClassName(restCategory.getName()) + "CommandOptions"); } } } - Set imports = new TreeSet<>(); for (RestEndpoint restEndpoint : restCategory.getEndpoints()) { if (isValidImport(restEndpoint.getResponseClass())) { imports.add(restEndpoint.getResponseClass().replaceAll("\\$", "\\.")); @@ -99,7 +98,7 @@ protected String getClassImports(String key) { if (bodyParam.isComplex() && !bodyParam.isCollection()) { if (bodyParam.getTypeClass() != null) { if (bodyParam.getTypeClass().contains("$")) { - imports.add(bodyParam.getTypeClass().substring(0, bodyParam.getTypeClass().lastIndexOf("$")) + ";"); + imports.add(bodyParam.getTypeClass().substring(0, bodyParam.getTypeClass().lastIndexOf("$"))); } else { imports.add(bodyParam.getTypeClass().replaceAll("\\$", "\\.")); } @@ -115,8 +114,17 @@ protected String getClassImports(String key) { } } + StringBuilder sb = new StringBuilder(); + sb.append("package ").append(config.getOptions().getExecutorsPackage()).append(";\n\n"); + for (String string : imports) { - sb.append("import ").append(string).append("\n"); + if (string.endsWith(";")) { + string = string.substring(0, string.length() - 1); + if (imports.contains(string)) { + continue; + } + } + sb.append("import ").append(string).append(";\n"); } sb.append("\n"); @@ -170,8 +178,7 @@ protected String getClassHeader(String key) { CategoryConfig config = availableCategoryConfigs.get(key); sb.append("public class " + getAsClassName(restCategory.getName()) + "CommandExecutor extends " + getExtendedClass() + " {\n\n"); - sb.append(" public " + getAsClassName(restCategory.getName()) + "CommandOptions " - + getAsVariableName(getAsCamelCase(restCategory.getName())) + "CommandOptions;\n\n"); + sb.append(" public " + getCommandOptionsClassName(restCategory) + " " + getCommandOptionsVarName(restCategory) + ";\n\n"); sb.append(" public " + getAsClassName(restCategory.getName()) + "CommandExecutor(" + getAsClassName(restCategory.getName()) + "CommandOptions " + getAsVariableName(getAsCamelCase(restCategory.getName())) + "CommandOptions) throws CatalogAuthenticationException {\n"); @@ -203,19 +210,24 @@ private String methodExecute(RestCategory restCategory, CategoryConfig categoryC } // } } - /* if (CollectionUtils.isNotEmpty(categoryConfig.getAddedMethods())) { - for (String methodName : categoryConfig.getAddedMethods()) { - sb.append(" case \"" + methodName + "\":\n"); - if (categoryConfig.getCommand(methodName) != null && StringUtils.isNotEmpty(categoryConfig.getCommand(methodName).getExecutorExtendedClassName())) { - sb.append(" " + categoryConfig.getCommand(methodName).getExecutorExtendedClassName() + " custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new " + categoryConfig.getCommand(methodName).getExecutorExtendedClassName() + "();\n"); - } else { - sb.append(" Parent" + getAsClassName(restCategory.getName()) + "CommandExecutor custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new Parent" + getAsClassName(restCategory.getName()) + "CommandExecutor();\n"); + if (CollectionUtils.isNotEmpty(categoryConfig.getAddedMethods())) { + for (String commandName : categoryConfig.getAddedMethods()) { - } - sb.append(" queryResponse = custom" + getAsClassName(restCategory.getName()) + "CommandExecutor." + getAsCamelCase(methodName) + "();\n"); + String executorClassName = getExecutorClassName(restCategory, categoryConfig, commandName); + String executorVarName = getExecutorVarName(restCategory); + String javaMethodName = getJavaMethodName(categoryConfig, commandName); + + String commandOptions = getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(categoryConfig, commandName); + String commonOptionsParams = getCommandOptionsVarName(restCategory) + ".commonCommandOptions.params"; + + sb.append(" case \"" + commandName + "\":\n"); + sb.append(" " + executorClassName + " " + executorVarName + " = new " + executorClassName + "(\n"); + sb.append(" new ObjectMap(" + commonOptionsParams + "), token,\n"); + sb.append(" clientConfiguration, getSessionManager(), appHome, getLogger());\n"); + sb.append(" queryResponse = " + executorVarName + "." + javaMethodName + "(" + commandOptions + ");\n"); sb.append(" break;\n"); } - }*/ + } sb.append(" default:\n"); sb.append(" logger.error(\"Subcommand not valid\");\n"); sb.append(" break;\n"); @@ -243,19 +255,14 @@ protected String getClassMethods(String key) { sb.append("\n"); sb.append(" private RestResponse<" + getValidResponseNames(restEndpoint.getResponse()) + "> " + getJavaMethodName(categoryConfig, commandName) + "() throws Exception {\n"); - sb.append(" logger.debug(\"Executing " + getAsCamelCase(commandName) + " in " + sb.append(" logger.debug(\"Executing " + getJavaMethodName(categoryConfig, commandName) + " in " + restCategory.getName() + " command line\");\n\n"); - if (categoryConfig.isExtendedOptionCommand(commandName)) { - sb.append(" Custom" + getAsClassName(restCategory.getName()) + "CommandOptions." + getAsClassName(getAsCamelCase(commandName)) - + "CommandOptions commandOptions = " + getAsVariableName(getAsCamelCase(restCategory.getName())) + - "CommandOptions." - + getAsCamelCase(commandName) + "CommandOptions;\n"); - } else { - sb.append(" " + getAsClassName(restCategory.getName()) + "CommandOptions." + getAsClassName(getAsCamelCase(commandName)) - + "CommandOptions commandOptions = " + getAsVariableName(getAsCamelCase(restCategory.getName())) + - "CommandOptions." - + getAsCamelCase(commandName) + "CommandOptions;\n"); - } + + String commandOptionsClassName = getCommandOptionsClassName(restCategory, categoryConfig, commandName) + "." + getSubCommandOptionsClassName(categoryConfig, commandName); + String commandOptionsLocalVarName = "commandOptions"; + String commandOptionsVarName = getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(categoryConfig, commandName); + + sb.append(" " + commandOptionsClassName + " " + commandOptionsLocalVarName + " = " + commandOptionsVarName + ";\n"); if (categoryConfig.isExecutorExtendedCommand(commandName)) { List body = new ArrayList<>(); for (RestParameter restParameter : restEndpoint.getParameters()) { @@ -306,13 +313,11 @@ protected String getClassMethods(String key) { } } - if (StringUtils.isNotEmpty(categoryConfig.getCommand(commandName).getExecutorExtendedClassName())) { - sb.append(" " + categoryConfig.getCommand(commandName).getExecutorExtendedClassName() + " custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new " + categoryConfig.getCommand(commandName).getExecutorExtendedClassName() + "(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());\n"); - } else { - sb.append(" Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor custom" + getAsClassName(restCategory.getName()) + "CommandExecutor = new Custom" + getAsClassName(restCategory.getName()) + "CommandExecutor(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());\n"); - } + String executorClassName = getExecutorClassName(restCategory, categoryConfig, commandName); + String executorVarName = getExecutorVarName(restCategory); - sb.append(" return custom" + getAsClassName(restCategory.getName()) + "CommandExecutor." + getAsCamelCase(commandName) + "();\n"); + sb.append(" " + executorClassName + " " + executorVarName + " = new " + executorClassName + "(queryParams, token, clientConfiguration, getSessionManager(), appHome, getLogger());\n"); + sb.append(" return " + executorVarName + "." + getJavaMethodName(categoryConfig, commandName) + "(commandOptions);\n"); } else { sb.append(getQueryParams(restEndpoint, categoryConfig, commandName)); sb.append(getBodyParams(restCategory, restEndpoint, categoryConfig, commandName)); @@ -351,15 +356,6 @@ private String getReturn(RestCategory restCategory, RestEndpoint restEndpoint, C return res; } - private String getJavaMethodName(CategoryConfig config, String commandName) { - Command command = config.getCommand(commandName); - String commandMethod = getAsCamelCase(commandName); - if (command != null && StringUtils.isNotEmpty(command.getRename())) { - commandMethod = command.getRename(); - } - return commandMethod; - } - private String getBodyParams(RestCategory restCategory, RestEndpoint restEndpoint, CategoryConfig config, String commandName) { StringBuilder sb = new StringBuilder(); String bodyClassName = restEndpoint.getBodyClassName(); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java index 80c9a5fcdd7..bfa784319b7 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/OptionsCliRestApiWriter.java @@ -114,15 +114,16 @@ protected String getClassHeader(String key) { String commandName = getCommandName(restCategory, restEndpoint); // if ("POST".equals(restEndpoint.getMethod()) || restEndpoint.hasParameters()) { if (config.isAvailableCommand(commandName)) { - sb.append(" public " + getAsClassName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions " - + getAsVariableName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions;\n"); + String className = getSubCommandOptionsClassName(config, commandName); + String varName = getSubCommandOptionsVarName(config, commandName); + sb.append(" public " + className + " " + varName + ";\n"); } // } } sb.append("\n"); sb.append("\n"); - sb.append(" public " + getAsClassName(restCategory.getName()) + "CommandOptions(CommonCommandOptions commonCommandOptions, " + + sb.append(" public " + getCommandOptionsClassName(restCategory) + "(CommonCommandOptions commonCommandOptions, " + "JCommander jCommander) {\n"); sb.append(" \n"); if (config.isOptionExtended()) { @@ -135,9 +136,9 @@ protected String getClassHeader(String key) { String commandName = getCommandName(restCategory, restEndpoint); // if ("POST".equals(restEndpoint.getMethod()) || restEndpoint.hasParameters()) { if (config.isAvailableCommand(commandName)) { - sb.append(" this." + getAsVariableName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions = " + - "new " - + getAsClassName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions();\n"); + String varName = getSubCommandOptionsVarName(config, commandName); + String className = getSubCommandOptionsClassName(config, commandName); + sb.append(" this." + varName + " = " + "new " + className + "();\n"); } // } } @@ -158,7 +159,7 @@ protected String getClassMethods(String key) { if (config.isAvailableCommand(commandName) && !config.isExtendedOptionCommand(commandName)) { sb.append(" @Parameters(commandNames = {\"" + reverseCommandName(commandName) + "\"}, commandDescription =\"" + restEndpoint.getDescription().replaceAll("\"", "'") + "\")\n"); - sb.append(" public class " + getAsClassName(getAsCamelCase(getMethodName(restCategory, restEndpoint))) + "CommandOptions " + + sb.append(" public class " + getSubCommandOptionsClassName(config, commandName) + " " + "{\n"); sb.append(" \n"); sb.append(" @ParametersDelegate\n"); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java index 828d12be889..dd4717b0fee 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/cli/ParserCliRestApiWriter.java @@ -116,17 +116,15 @@ protected String getClassHeader(String key) { // if ("POST".equals(restEndpoint.getMethod()) || restEndpoint.hasParameters()) { if (config.isAvailableCommand(commandName)) { sb.append(" " + getAsVariableName(restCategory.getName()) + "SubCommands.addCommand(\"" + reverseCommandName(commandName) + "\", " - + getAsVariableName(restCategory.getName()) + "CommandOptions." + getAsCamelCase(commandName) + - "CommandOptions);\n"); + + getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(config, commandName) + ");\n"); } // } } if (CollectionUtils.isNotEmpty(config.getAddedMethods())) { - for (String methodName : config.getAddedMethods()) { - sb.append(" " + getAsVariableName(restCategory.getName()) + "SubCommands.addCommand(\"" + methodName + "\", " - + getAsVariableName(restCategory.getName()) + "CommandOptions." + getAsCamelCase(methodName) + - "CommandOptions);\n"); + for (String commandName : config.getAddedMethods()) { + sb.append(" " + getAsVariableName(restCategory.getName()) + "SubCommands.addCommand(\"" + commandName + "\", " + + getCommandOptionsVarName(restCategory) + "." + getSubCommandOptionsVarName(config, commandName) + ");\n"); } } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java index 528853faeb6..edb1ab223c1 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/MetaWSServer.java @@ -17,8 +17,6 @@ package org.opencb.opencga.server.rest; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; -import org.opencb.commons.datastore.core.Event; import org.opencb.commons.utils.DataModelsUtils; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.exceptions.VersionException; @@ -26,6 +24,7 @@ import org.opencb.opencga.core.tools.annotations.Api; import org.opencb.opencga.core.tools.annotations.ApiOperation; import org.opencb.opencga.core.tools.annotations.ApiParam; +import org.opencb.opencga.server.OpenCGAHealthCheckMonitor; import org.opencb.opencga.server.generator.RestApiParser; import org.opencb.opencga.server.generator.models.RestApi; import org.opencb.opencga.server.rest.admin.AdminWSServer; @@ -44,13 +43,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import java.io.IOException; -import java.time.Duration; -import java.time.LocalTime; -import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; /** * Created by pfurio on 05/05/17. @@ -60,15 +53,6 @@ @Api(value = "Meta", description = "Meta RESTful Web Services API") public class MetaWSServer extends OpenCGAWSServer { - private static final AtomicReference healthCheckErrorMessage = new AtomicReference<>(); - private static final AtomicReference lastAccess = new AtomicReference<>(LocalTime.now()); - private static final Map healthCheckResults = new ConcurrentHashMap<>(); - private final String OKAY = "OK"; - private final String NOT_OKAY = "KO"; - private final String SOLR = "Solr"; - private final String VARIANT_STORAGE = "VariantStorage"; - private final String CATALOG_MONGO_DB = "CatalogMongoDB"; - public MetaWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { super(uriInfo, httpServletRequest, httpHeaders); @@ -111,116 +95,16 @@ public Response fail() { @Path("/status") @ApiOperation(httpMethod = "GET", value = "Database status.", response = Map.class) public Response status() { + OpenCGAResult queryResult = healthCheckMonitor.getStatus(); + OpenCGAHealthCheckMonitor.HealthCheckStatus status = queryResult.first(); - OpenCGAResult> queryResult = new OpenCGAResult<>(); - StopWatch stopWatch = StopWatch.createStarted(); - - if (shouldUpdateStatus()) { - logger.debug("Update HealthCheck cache status"); - updateHealthCheck(); - } else { - logger.debug("HealthCheck results from cache at " + lastAccess.get().format(DateTimeFormatter.ofPattern("HH:mm:ss"))); - queryResult.setEvents(Collections.singletonList(new Event(Event.Type.INFO, "HealthCheck results from cache at " - + lastAccess.get().format(DateTimeFormatter.ofPattern("HH:mm:ss"))))); - } - - queryResult.setTime(((int) stopWatch.getTime(TimeUnit.MILLISECONDS))); - queryResult.setResults(Collections.singletonList(healthCheckResults)); - - if (isHealthy()) { - logger.debug("HealthCheck : " + healthCheckResults.toString()); + if (status.isHealthy()) { + logger.debug("HealthCheck : " + status); return createOkResponse(queryResult); } else { - logger.error("HealthCheck : " + healthCheckResults.toString()); - return createErrorResponse(healthCheckErrorMessage.get(), queryResult); - } - } - - private boolean shouldUpdateStatus() { - if (!isHealthy()) { - // Always update if not healthy - return true; - } - // If healthy, only update every "healthCheck.interval" seconds - long elapsedTime = Duration.between(lastAccess.get(), LocalTime.now()).getSeconds(); - return elapsedTime > configuration.getHealthCheck().getInterval(); - } - - private synchronized void updateHealthCheck() { - if (!shouldUpdateStatus()) { - // Skip update! - return; - } - StringBuilder errorMsg = new StringBuilder(); - - Map newHealthCheckResults = new HashMap<>(); - newHealthCheckResults.put(CATALOG_MONGO_DB, ""); - newHealthCheckResults.put(VARIANT_STORAGE, ""); - newHealthCheckResults.put(SOLR, ""); - - StopWatch totalTime = StopWatch.createStarted(); - StopWatch catalogMongoDBTime = StopWatch.createStarted(); - try { - if (catalogManager.getCatalogDatabaseStatus()) { - newHealthCheckResults.put(CATALOG_MONGO_DB, OKAY); - } else { - newHealthCheckResults.put(CATALOG_MONGO_DB, NOT_OKAY); - } - } catch (Exception e) { - newHealthCheckResults.put(CATALOG_MONGO_DB, NOT_OKAY); - errorMsg.append(e.getMessage()); - } - catalogMongoDBTime.stop(); - - StopWatch storageTime = StopWatch.createStarted(); - try { - storageEngineFactory.getVariantStorageEngine(null, configuration.getDatabasePrefix() + "_test_connection", "test_connection") - .testConnection(); - newHealthCheckResults.put("VariantStorageId", storageEngineFactory.getVariantStorageEngine().getStorageEngineId()); - newHealthCheckResults.put(VARIANT_STORAGE, OKAY); - } catch (Exception e) { - newHealthCheckResults.put(VARIANT_STORAGE, NOT_OKAY); - errorMsg.append(e.getMessage()); -// errorMsg.append(" No storageEngineId is set in configuration or Unable to initiate storage Engine, ").append(e.getMessage() -// ).append(", "); - } - storageTime.stop(); - - StopWatch solrEngineTime = StopWatch.createStarted(); - if (storageEngineFactory.getStorageConfiguration().getSearch().isActive()) { - try { - if (variantManager.isSolrAvailable()) { - newHealthCheckResults.put(SOLR, OKAY); - } else { - errorMsg.append(", unable to connect with solr, "); - newHealthCheckResults.put(SOLR, NOT_OKAY); - } - } catch (Exception e) { - newHealthCheckResults.put(SOLR, NOT_OKAY); - errorMsg.append(e.getMessage()); - } - } else { - newHealthCheckResults.put(SOLR, "solr not active in storage-configuration!"); + logger.error("HealthCheck : " + status); + return createErrorResponse(status.getErrorMessage(), queryResult); } - solrEngineTime.stop(); - - if (totalTime.getTime(TimeUnit.SECONDS) > 5) { - logger.warn("Slow OpenCGA status: Updated time: {}. Catalog: {} , Storage: {} , Solr: {}", - totalTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, - catalogMongoDBTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, - storageTime.getTime(TimeUnit.MILLISECONDS) / 1000.0, - solrEngineTime.getTime(TimeUnit.MILLISECONDS) / 1000.0 - ); - } - - if (errorMsg.length() == 0) { - healthCheckErrorMessage.set(null); - } else { - healthCheckErrorMessage.set(errorMsg.toString()); - } - - healthCheckResults.putAll(newHealthCheckResults); - lastAccess.set(LocalTime.now()); } @GET @@ -270,8 +154,4 @@ public Response api(@ApiParam(value = "List of categories to get API from") @Que RestApi restApi = new RestApiParser().parse(classes, summary); return createOkResponse(new OpenCGAResult<>(0, Collections.emptyList(), 1, Collections.singletonList(restApi.getCategories()), 1)); } - - private boolean isHealthy() { - return healthCheckResults.isEmpty() ? false : !healthCheckResults.values().stream().anyMatch(x -> x.equals(NOT_OKAY)); - } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java index 03a39102d9e..761c0a3bd52 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/OpenCGAWSServer.java @@ -53,6 +53,7 @@ import org.opencb.opencga.core.response.RestResponse; import org.opencb.opencga.core.tools.ToolParams; import org.opencb.opencga.core.tools.annotations.ApiParam; +import org.opencb.opencga.server.OpenCGAHealthCheckMonitor; import org.opencb.opencga.server.WebServiceException; import org.opencb.opencga.server.rest.analysis.ClinicalWebService; import org.opencb.opencga.storage.core.StorageEngineFactory; @@ -137,6 +138,8 @@ public static Configuration getConfiguration() { protected static StorageEngineFactory storageEngineFactory; protected static VariantStorageManager variantManager; + protected static OpenCGAHealthCheckMonitor healthCheckMonitor; + private static final int DEFAULT_LIMIT = AbstractManager.DEFAULT_LIMIT; private static final int MAX_LIMIT = AbstractManager.MAX_LIMIT; private static final int MAX_ID_SIZE = 100; @@ -294,6 +297,8 @@ private static void initOpenCGAObjects() { catalogManager = new CatalogManager(configuration); storageEngineFactory = StorageEngineFactory.get(storageConfiguration); variantManager = new VariantStorageManager(catalogManager, storageEngineFactory); + healthCheckMonitor = new OpenCGAHealthCheckMonitor(configuration, catalogManager, storageEngineFactory, variantManager); + healthCheckMonitor.asyncUpdate(); MigrationSummary migrationSummary = catalogManager.getMigrationManager().getMigrationSummary(); if (migrationSummary.getMigrationsToBeApplied() > 0) { @@ -304,7 +309,6 @@ private static void initOpenCGAObjects() { } } } - } catch (Exception e) { errorMessage = e.getMessage(); // e.printStackTrace(); diff --git a/opencga-server/src/main/resources/cli-config.yaml b/opencga-server/src/main/resources/cli-config.yaml index b3a3c302120..4e5c24c5e9d 100644 --- a/opencga-server/src/main/resources/cli-config.yaml +++ b/opencga-server/src/main/resources/cli-config.yaml @@ -97,6 +97,9 @@ apiConfig: - name: top executorExtended: True optionExtended: True + - name: log + executorExtended: True + optionExtended: True addedMethods: - log - name: samples @@ -147,6 +150,7 @@ apiConfig: key: DiseasePanel commands: - name: import + ## Rename needed as `import` is a reserved word in Java rename: importPanels - name: analysisAlignment commandName: alignments diff --git a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java index 2550d6f0592..398b7f775cd 100644 --- a/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java +++ b/opencga-storage/opencga-storage-app/src/main/java/org/opencb/opencga/storage/app/cli/client/options/StorageVariantCommandOptions.java @@ -94,6 +94,9 @@ public static class GenericVariantIndexOptions { @Parameter(names = {"--load"}, description = "If present only the load stage is executed, transformation is skipped") public boolean load; + @Parameter(names = {"--force-reload"}, description = "If the file is already loaded, force a file reload") + public boolean forceReload; + @Parameter(names = {"--merge"}, description = "Currently two levels of merge are supported: \"basic\" mode merge genotypes of the same variants while \"advanced\" merge multiallelic and overlapping variants.") public String merge; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java index 3c193134518..536d8795d70 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/SampleMetadata.java @@ -18,6 +18,13 @@ */ public class SampleMetadata extends StudyResourceMetadata { + /** + * List of fileIds that contains the sample. + * This list is sorted by the order files were loaded. + * Only files of type {@link FileMetadata.Type#NORMAL} and {@link FileMetadata.Type#VIRTUAL} will be present in this list. + * If the sample was part of a set of {@link FileMetadata.Type#PARTIAL} files, only the + * {@link FileMetadata.Type#VIRTUAL} file will appear in this list. + */ private List files; private Set cohorts; // Prepared to have more than one secondary index per sample. diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java index 07b111f24bf..75a7c58b1d8 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/StudyResourceMetadata.java @@ -10,7 +10,7 @@ * * @author Jacobo Coll <jacobo167@gmail.com> */ -public abstract class StudyResourceMetadata { +public abstract class StudyResourceMetadata> { private int studyId; private int id; diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java index 2c12a0021e0..82c85135b74 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/metadata/models/Trio.java @@ -6,21 +6,42 @@ import java.util.List; import java.util.Objects; +/** + * This class represents a family trio. + * + * - Father and mother can be null + * - Child cannot be null + * - All samples must be unique + * - All samples must be different to NA (dash) + * + */ public class Trio { + public static final String NA = "-"; private final String id; private final String father; private final String mother; private final String child; + public Trio(String trio) { + String[] split = trio.split(","); + if (split.length != 3) { + throw new IllegalArgumentException("Expected three samples in trio '" + trio + "'"); + } + this.id = null; + this.child = split[0]; + this.father = NA.equals(split[1]) ? null : split[1]; + this.mother = NA.equals(split[2]) ? null : split[2]; + } + public Trio(List trio) { this(null, trio); } public Trio(String id, List trio) { this.id = id; - this.father = trio.get(1); - this.mother = trio.get(2); this.child = trio.get(0); + this.father = NA.equals(trio.get(1)) ? null : trio.get(1); + this.mother = NA.equals(trio.get(2)) ? null : trio.get(2); } public Trio(String father, String mother, String child) { @@ -29,8 +50,8 @@ public Trio(String father, String mother, String child) { public Trio(String id, String father, String mother, String child) { this.id = id; - this.father = father; - this.mother = mother; + this.father = NA.equals(father) ? null : father; + this.mother = NA.equals(mother) ? null : mother; this.child = child; } @@ -50,6 +71,11 @@ public String getChild() { return child; } + /** + * Returns a list with the non-null samples in the trio. + * + * @return List of samples + */ public List toList() { ArrayList list = new ArrayList<>(3); list.add(getChild()); @@ -82,6 +108,31 @@ public int hashCode() { return Objects.hash(id, father, mother, child); } + /** + * Serialize the trio into a string contain the three samples separated by commas. + * order: child, father, mother. + * If the father or mother are null, they will be replaced by {@link #NA}. + * + * Can be deserialized using {@link #Trio(String)}. + * + * @return String + */ + public String serialize() { + ArrayList list = new ArrayList<>(3); + list.add(getChild()); + if (getFather() == null) { + list.add(NA); + } else { + list.add(getFather()); + } + if (getMother() == null) { + list.add(NA); + } else { + list.add(getMother()); + } + return Strings.join(list, ','); + } + @Override public String toString() { return Strings.join(toList(), ','); diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java index a4f560e49f2..a3ab7e539b5 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStorageEngine.java @@ -140,11 +140,11 @@ public enum SplitData { REGION, MULTI; - public static boolean isPartial(ObjectMap options) { - return isPartial(from(options)); + public static boolean isPartialSplit(ObjectMap options) { + return isPartialSplit(from(options)); } - public static boolean isPartial(SplitData splitData) { + public static boolean isPartialSplit(SplitData splitData) { return splitData == CHROMOSOME || splitData == REGION; } @@ -582,45 +582,52 @@ public void sampleIndexAnnotate(String study, List samples, ObjectMap op * The Family Index is used alongside with the SampleIndex to speed up queries involving children and parents. * * @param study Study - * @param trios List of trios "father, mother, child". - * Missing parents in trios are specified with "-", - * If a family has two children, two trios should be defined. + * @param trios Trios to index. If a family has two children, two trios should be defined. * @param options Other options * @throws StorageEngineException in an error occurs * @return List of trios used to index. Empty if there was nothing to do. */ - public DataResult> familyIndex(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult familyIndex(String study, List trios, ObjectMap options) throws StorageEngineException { throw new UnsupportedOperationException("Unsupported familyIndex"); } - public DataResult> familyIndexUpdate(String study, ObjectMap options) throws StorageEngineException { + /** + * Update the family index. + * The Family Index is used alongside with the SampleIndex to speed up queries involving children and parents. + * + * @param study Study + * @param options Other options + * @throws StorageEngineException in an error occurs + * @return List of trios used to index. Empty if there was nothing to do. + */ + public DataResult familyIndexUpdate(String study, ObjectMap options) throws StorageEngineException { StudyMetadata studyMetadata = getMetadataManager().getStudyMetadata(study); int studyId = studyMetadata.getId(); int version = studyMetadata.getSampleIndexConfigurationLatest().getVersion(); - List> trios = new LinkedList<>(); + List trios = new LinkedList<>(); for (SampleMetadata sampleMetadata : getMetadataManager().sampleMetadataIterable(studyId)) { if (sampleMetadata.isFamilyIndexDefined()) { if (sampleMetadata.getFamilyIndexStatus(version) != TaskMetadata.Status.READY) { // This sample's family index needs to be updated String father; if (sampleMetadata.getFather() == null) { - father = "-"; + father = null; } else { father = getMetadataManager().getSampleName(studyId, sampleMetadata.getFather()); } String mother; if (sampleMetadata.getMother() == null) { - mother = "-"; + mother = null; } else { mother = getMetadataManager().getSampleName(studyId, sampleMetadata.getMother()); } - trios.add(Arrays.asList(father, mother, sampleMetadata.getName())); + trios.add(new Trio(father, mother, sampleMetadata.getName())); } } } if (trios.isEmpty()) { logger.info("Nothing to do!"); - return new DataResult>().setEvents(Collections.singletonList(new Event(Event.Type.INFO, "Nothing to do"))); + return new DataResult().setEvents(Collections.singletonList(new Event(Event.Type.INFO, "Nothing to do"))); } else { return familyIndex(study, trios, options); } @@ -1025,6 +1032,7 @@ protected void postRemoveFiles(String study, List fileIds, List fields, QueryOptions options } public DataResult count(Query query) throws StorageEngineException { - VariantQueryResult result = get(query, new QueryOptions(QueryOptions.INCLUDE, VariantField.ID) - .append(QueryOptions.LIMIT, 1) - .append(QueryOptions.COUNT, true)); + VariantQueryResult result = get(VariantQueryUtils.copy(query), + new QueryOptions(QueryOptions.INCLUDE, VariantField.ID) + .append(QueryOptions.LIMIT, 1) + .append(QueryOptions.COUNT, true)); return new DataResult<>( result.getTime(), result.getEvents(), diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java index 44c063cf17b..19b0631659f 100644 --- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/VariantStoragePipeline.java @@ -52,9 +52,9 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.common.YesNoAuto; +import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.common.mixins.GenericRecordAvroJsonMixin; import org.opencb.opencga.storage.core.StoragePipeline; -import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.io.plain.StringDataReader; @@ -172,7 +172,17 @@ public URI preTransform(URI input) throws StorageEngineException, IOException, F } return existingStudyMetadata; }); - if (VariantStorageEngine.SplitData.isPartial(options) + if (options.getBoolean(FORCE.key())) { + Integer fileId = getMetadataManager().getFileId(studyMetadata.getId(), fileName, true); + if (fileId != null) { + // File is indexed. Mark as non indexed. + getMetadataManager().updateFileMetadata(studyMetadata.getId(), fileId, fileMetadata -> { + fileMetadata.setIndexStatus(TaskMetadata.Status.NONE); + }); + logger.info("File '{}' already loaded. Force reload!", fileName); + } + } + if (VariantStorageEngine.SplitData.isPartialSplit(options) && !options.getString(LOAD_VIRTUAL_FILE.key(), "").isEmpty()) { setFileId(smm.registerPartialFile(studyMetadata.getId(), input.getPath())); } else { @@ -549,7 +559,7 @@ protected void preLoadRegisterAndValidateFile(int studyId, VariantFileMetadata f final int fileId; String virtualFile = options.getString(LOAD_VIRTUAL_FILE.key()); - if (VariantStorageEngine.SplitData.isPartial(options)) { + if (VariantStorageEngine.SplitData.isPartialSplit(options)) { if (StringUtils.isEmpty(virtualFile)) { fileId = getMetadataManager().registerFile(studyId, fileMetadata); // throw new StorageEngineException("Unable to load file with 'split-data'. Missing virtual file belonging! " diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java index e7ceeee7970..fe5313bdc80 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageBaseTest.java @@ -17,8 +17,11 @@ package org.opencb.opencga.storage.core.variant; import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Test; import org.junit.*; import org.junit.rules.ExpectedException; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; import org.opencb.biodata.formats.io.FileFormatException; import org.opencb.biodata.models.variant.Variant; import org.opencb.commons.datastore.core.ObjectMap; @@ -37,6 +40,7 @@ import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; @@ -115,6 +119,10 @@ public abstract class VariantStorageBaseTest extends GenericTest implements Vari @Rule public ExpectedException thrown = ExpectedException.none(); + @ClassRule + public static TestClassNameWatcher testClassNameWatcher = new TestClassNameWatcher(); + + @BeforeClass public static void _beforeClass() throws Exception { // System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "debug"); @@ -191,7 +199,7 @@ public static URI getPlatinumFile(int fileId) throws IOException { private static void newRootDir() throws IOException { - rootDir = Paths.get("target/test-data", "junit-opencga-storage-" + TimeUtils.getTimeMillis() + "_" + RandomStringUtils.randomAlphabetic(3)); + rootDir = Paths.get("target/test-data", "junit-" + testClassNameWatcher.getTestClassSimpleName() + "-" + TimeUtils.getTimeMillis() + "_" + RandomStringUtils.randomAlphabetic(3)); Files.createDirectories(rootDir); } @@ -199,21 +207,33 @@ public static void setRootDir(Path rootDir) { VariantStorageBaseTest.rootDir = rootDir; } - public static URI newOutputUri() throws IOException { - return newOutputUri(1, outputUri); - } - - public static URI newOutputUri(int extraCalls) throws IOException { - return newOutputUri(1 + extraCalls, outputUri); - } - - public static URI newOutputUri(int extraCalls, URI outputUri) throws IOException { + protected static URI newOutputUri() throws IOException { + String dirName = null; StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); - // stackTrace[0] = "Thread.currentThread" - // stackTrace[1] = "newOutputUri" - // stackTrace[2] = caller method - String testName = stackTrace[2 + extraCalls].getMethodName(); - return newOutputUri(testName, outputUri); + for (int i = 0; i < stackTrace.length; i++) { + StackTraceElement element = stackTrace[i]; + try { + Class aClass = Class.forName(element.getClassName()); + Method method = aClass.getMethod(element.getMethodName()); + if (method.isAnnotationPresent(Test.class) + || method.isAnnotationPresent(Before.class) + || method.isAnnotationPresent(BeforeClass.class) + || method.isAnnotationPresent(After.class) + || method.isAnnotationPresent(AfterClass.class)) { + dirName = element.getMethodName(); + break; + } + } catch (ClassNotFoundException | NoSuchMethodException ignore) { + } + } + if (dirName == null) { + // stackTrace[0] = "Thread.currentThread" + // stackTrace[1] = "newOutputUri" + // stackTrace[2] = caller method + dirName = stackTrace[2].getMethodName(); + } + + return newOutputUri(dirName, outputUri); } protected static URI newOutputUri(String testName) throws IOException { @@ -434,4 +454,18 @@ public void printActiveThreads() { } System.out.println("========================================="); } + + private static class TestClassNameWatcher extends TestWatcher { + + private String testClass; + + @Override + protected void starting(Description d) { + this.testClass = d.getTestClass().getSimpleName(); + } + + public String getTestClassSimpleName() { + return this.testClass; + } + } } diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java index c81c7d21831..55866e24160 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/dummy/DummyVariantStorageEngine.java @@ -28,6 +28,7 @@ import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.io.VariantImporter; @@ -102,7 +103,7 @@ public DummyVariantStoragePipeline newStoragePipeline(boolean connected) throws } @Override - public DataResult> familyIndex(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult familyIndex(String study, List trios, ObjectMap options) throws StorageEngineException { logger.info("Running family index!"); VariantStorageMetadataManager metadataManager = getMetadataManager(); StudyMetadata studyMetadata = metadataManager.getStudyMetadata(study); @@ -121,7 +122,7 @@ public DataResult> familyIndex(String study, List> tri } }); } - return new DataResult>().setResults(trios); + return new DataResult().setResults(trios); } @Override diff --git a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java index 031360d3298..7ae1991221d 100644 --- a/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java +++ b/opencga-storage/opencga-storage-core/src/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java @@ -21,20 +21,22 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; import org.apache.solr.core.NodeConfig; -import org.apache.solr.core.SolrResourceLoader; import org.junit.rules.ExternalResource; import org.opencb.commons.datastore.solr.SolrManager; import org.opencb.opencga.core.common.GitRepositoryState; +import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.search.solr.VariantSearchManager; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.*; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.DB_NAME; +import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri; /** * + @@ -61,18 +63,19 @@ public VariantSolrExternalResource(boolean embeded) { protected void before() throws Throwable { super.before(); - Path rootDir = getTmpRootDir(); + Path rootDir = Paths.get("target/test-data", "junit-variant-solr-" + TimeUtils.getTimeMillis()); + Files.createDirectories(rootDir); String configSet = "opencga-variant-configset-" + GitRepositoryState.getInstance().getBuildVersion(); // Copy configuration - getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml"); - getResourceUri("managed-schema", "configsets/" + configSet + "/managed-schema"); - getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json"); - getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt"); - getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt"); - getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt"); - getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt"); + getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml", rootDir); + getResourceUri("managed-schema", "configsets/" + configSet + "/managed-schema", rootDir); + getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json", rootDir); + getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt", rootDir); + getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt", rootDir); + getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt", rootDir); String solrHome = rootDir.resolve("solr").toString(); diff --git a/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz new file mode 100644 index 00000000000..798a4bf531f Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz new file mode 100644 index 00000000000..2206202e390 Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz differ diff --git a/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz new file mode 100644 index 00000000000..a2557925cdd Binary files /dev/null and b/opencga-storage/opencga-storage-core/src/test/resources/giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz differ diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java index 063219f7654..5c3d0e7a0d1 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java @@ -513,49 +513,44 @@ public URI postLoad(URI input, URI output) throws StorageEngineException { metadataManager.setStatus(getStudyId(), taskId, Status.READY); boolean loadSampleIndex = YesNoAuto.parse(getOptions(), LOAD_SAMPLE_INDEX.key()).orYes().booleanValue(); + int updatedSamples = 0; for (Integer sampleId : metadataManager.getSampleIdsFromFileId(getStudyId(), getFileId())) { // Worth to check first to avoid too many updates in scenarios like 1000G SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), sampleId); boolean updateSampleIndexStatus = loadSampleIndex && sampleMetadata.getSampleIndexStatus(sampleIndexVersion) != Status.READY; int actualLargestVariantLength = sampleMetadata.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH); - boolean alreadyLoadedFiles = false; - if (sampleMetadata.getFiles().size() > 1) { - int loadedFiles = 0; - for (Integer fileId : sampleMetadata.getFiles()) { - if (metadataManager.isFileIndexed(getStudyId(), fileId)) { - loadedFiles++; - } - } - if (loadedFiles > 1) { - alreadyLoadedFiles = true; - } - metadataManager.getFileIdsFromSampleId(1, 1, true); - } + boolean isLargestVariantLengthDefined = sampleMetadata.getAttributes() + .containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH); + boolean unknownLargestVariantLength = sampleMetadata.getAttributes() + .getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH); + boolean updateLargestVariantLength; - if (alreadyLoadedFiles) { - if (actualLargestVariantLength > 0) { - // Already loaded files, with a valid value. Update if needed. - updateLargestVariantLength = largestVariantLength > actualLargestVariantLength; - } else { - // Already loaded files without a valid value. Do not set a value, as it might be smaller than previous files. - updateLargestVariantLength = false; - } + if (isLargestVariantLengthDefined) { + // Update only if the new value is bigger than the current one + updateLargestVariantLength = largestVariantLength > actualLargestVariantLength; + } else if (unknownLargestVariantLength) { + // Already loaded files with unknown largest variant length. Do not update value. + updateLargestVariantLength = false; } else { // First file loaded. Update value updateLargestVariantLength = true; } if (updateSampleIndexStatus || updateLargestVariantLength) { + updatedSamples++; metadataManager.updateSampleMetadata(getStudyId(), sampleId, s -> { if (updateSampleIndexStatus) { s.setSampleIndexStatus(Status.READY, sampleIndexVersion); } if (updateLargestVariantLength) { - s.getAttributes().put(SampleIndexSchema.LARGEST_VARIANT_LENGTH, largestVariantLength); + int current = s.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, largestVariantLength); + s.getAttributes().put(SampleIndexSchema.LARGEST_VARIANT_LENGTH, Math.max(current, largestVariantLength)); } }); } } + getLoadStats().put("updatedSampleMetadata", updatedSamples); + logger.info("Updated status of {} samples", updatedSamples); boolean loadArchive = YesNoAuto.parse(getOptions(), LOAD_ARCHIVE.key()).orYes().booleanValue(); if (loadArchive) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java index c3afc1f5040..917b679cf1c 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java @@ -46,10 +46,7 @@ import org.opencb.opencga.storage.core.io.managers.IOConnectorProvider; import org.opencb.opencga.storage.core.metadata.VariantMetadataFactory; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; -import org.opencb.opencga.storage.core.metadata.models.FileMetadata; -import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.*; import org.opencb.opencga.storage.core.utils.CellBaseUtils; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -400,7 +397,7 @@ public void sampleIndexAnnotate(String study, List samples, ObjectMap op @Override - public DataResult> familyIndex(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult familyIndex(String study, List trios, ObjectMap options) throws StorageEngineException { options = getMergedOptions(options); return new FamilyIndexLoader(getSampleIndexDBAdaptor(), getDBAdaptor(), getMRExecutor()) .load(study, trios, options); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java index a5a6e56ef14..4d3718a6250 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTask.java @@ -376,7 +376,7 @@ public boolean getOverlappingVariants(Variant variant, int fileId, List> overlappingRecords) { String chromosome = vcfSlice.getChromosome(); int position = vcfSlice.getPosition(); - Integer resetPosition = null; + Integer resetIteratorIndex = null; boolean isAlreadyPresent = false; int firstIndex = iterator.nextIndex(); // Assume sorted VcfRecords @@ -388,8 +388,8 @@ public boolean getOverlappingVariants(Variant variant, int fileId, String alternate = vcfRecord.getAlternate(); // If the VcfRecord starts after the variant, stop looking for variants if (overlapsWith(variant, chromosome, start, end)) { - if (resetPosition == null) { - resetPosition = Math.max(iterator.previousIndex() - 1, firstIndex); + if (resetIteratorIndex == null) { + resetIteratorIndex = Math.max(iterator.previousIndex() - 1, firstIndex); } // if (skipReferenceVariants && hasAllReferenceGenotype(vcfSlice, vcfRecord)) { @@ -406,8 +406,8 @@ public boolean getOverlappingVariants(Variant variant, int fileId, overlappingRecords.add(ImmutablePair.of(vcfSlice, vcfRecord)); } else if (isRegionAfterVariantStart(start, end, variant)) { - if (resetPosition == null) { - resetPosition = Math.max(iterator.previousIndex() - 1, firstIndex); + if (resetIteratorIndex == null) { + resetIteratorIndex = Math.max(iterator.previousIndex() - 1, firstIndex); } // Shouldn't happen that the first VcfRecord from the iterator is beyond the variant to process, // and is not the first VcfRecord from the slice. @@ -429,14 +429,15 @@ public boolean getOverlappingVariants(Variant variant, int fileId, } } } - if (resetPosition == null && !iterator.hasNext()) { + if (resetIteratorIndex == null && !iterator.hasNext()) { // If the iterator reaches the end without finding any point, reset the iterator - resetPosition = firstIndex; + resetIteratorIndex = firstIndex; } // Send back the iterator - if (resetPosition != null) { -// logger.info("Reset from " + iterator.nextIndex() + " to " + resetPosition + ". fileId : " + fileId + " variant " + variant); - while (iterator.nextIndex() > resetPosition) { + if (resetIteratorIndex != null) { +// logger.info("Reset from " + iterator.nextIndex() + " to " + resetIteratorIndex +// + ". fileId : " + fileId + " variant " + variant); + while (iterator.nextIndex() > resetIteratorIndex) { iterator.previous(); } } @@ -506,10 +507,19 @@ protected LinkedHashMap getSamplePosition(Integer fileId) { }); } + /** + * Checks if the given region is entirely or partially (overlapping) after the variant start. + * + * @param start Start of a region + * @param end End of a region (might be before the start if the region represented an insertion) + * @param variant Variant to check + * @return if the region is after the variant + */ public static boolean isRegionAfterVariantStart(int start, int end, Variant variant) { - int pos = Math.min(start, end); - int variantPos = Math.min(variant.getStart(), variant.getEnd()); - return pos > variantPos; + // Get region max position. In the region represents an insertion, the start might be after the end + int regionMaxPos = Math.max(start, end); + int variantMinPos = Math.min(variant.getStart(), variant.getEnd()); + return regionMaxPos > variantMinPos; } public static boolean overlapsWith(Variant variant, String chromosome, int start, int end) { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java index 5829a9580a8..510085420dd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/SampleIndexOnlyVariantQueryExecutor.java @@ -248,11 +248,7 @@ private boolean isIncludeCovered(SampleIndexQuery sampleIndexQuery, Query inputQ return false; } - List sampleDataKeys = VariantQueryUtils.getIncludeSampleData(inputQuery); - if (sampleDataKeys == null) { - // Undefined, get default sampleDataKeys - sampleDataKeys = HBaseToVariantConverter.getFixedFormat(study.getStudyMetadata()); - } + List sampleDataKeys = getSampleDataKeys(inputQuery, study); if (sampleDataKeys.size() != 1) { // One and only one sampledatakey @@ -490,8 +486,9 @@ private class AddMissingDataTask implements Task { for (Integer fileId : fileIds) { filesFromSample.add(metadataManager.getFileName(studyId, fileId)); } - List includeSampleData = VariantQueryUtils.getIncludeSampleData(parsedQuery.getInputQuery()); - gtIdx = includeSampleData.indexOf("GT"); + + List sampleDataKeys = getSampleDataKeys(parsedQuery.getInputQuery(), parsedQuery.getProjection().getStudy(studyId)); + gtIdx = sampleDataKeys.indexOf("GT"); } @Override @@ -657,4 +654,13 @@ private void mergeFileEntries(StudyEntry studyEntry, List newFileEntr } } } + + private List getSampleDataKeys(Query parsedQuery, VariantQueryProjection.StudyVariantQueryProjection parsedQuery1) { + List sampleDataKeys = VariantQueryUtils.getIncludeSampleData(parsedQuery); + if (sampleDataKeys == null) { + // Undefined, get default sampleDataKeys + sampleDataKeys = HBaseToVariantConverter.getFixedFormat(parsedQuery1.getStudyMetadata()); + } + return sampleDataKeys; + } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java index 36f17f5f3a1..ada52270419 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexDriver.java @@ -17,6 +17,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam; import org.opencb.opencga.storage.hadoop.variant.AbstractVariantsTableDriver; import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; @@ -98,28 +99,28 @@ protected void parseAndValidateParameters() throws IOException { if (StringUtils.isNotEmpty(triosStr)) { String[] trios = triosStr.split(";"); List trioList = new ArrayList<>(3); - for (String trio : trios) { - for (String sample : trio.split(",")) { - Integer sampleId; - if (sample.equals("-")) { - sampleId = MISSING_SAMPLE; - } else { - sampleId = metadataManager.getSampleId(getStudyId(), sample); - if (sampleId == null) { - throw new IllegalArgumentException("Sample '" + sample + "' not found."); - } - } - trioList.add(sampleId); + for (String trioString : trios) { + Trio trio = new Trio(Arrays.asList(trioString.split(","))); + + if (trio.getFather() == null) { + trioList.add(MISSING_SAMPLE); + } else { + trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getFather())); } - if (trioList.size() != 3) { - throw new IllegalArgumentException("Found trio with " + trioList.size() + " members, instead of 3: " + trioList); + if (trio.getMother() == null) { + trioList.add(MISSING_SAMPLE); + } else { + trioList.add(metadataManager.getSampleIdOrFail(getStudyId(), trio.getMother())); } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), trioList.get(2)); + int childId = metadataManager.getSampleIdOrFail(getStudyId(), trio.getChild()); + trioList.add(childId); + + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(getStudyId(), childId); if (!overwrite && sampleMetadata.getFamilyIndexStatus(sampleIndexVersion) == TaskMetadata.Status.READY) { LOGGER.info("Skip sample " + sampleMetadata.getName() + ". Already precomputed!"); } else { sampleIds.addAll(trioList); - LOGGER.info("Trio: " + trio + " -> " + trioList); + LOGGER.info("Trio: " + trioString + " -> " + trioList); } trioList.clear(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java index 244945ce71e..17083348c45 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexLoader.java @@ -9,6 +9,7 @@ import org.opencb.opencga.storage.core.metadata.models.CohortMetadata; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.TaskMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; @@ -17,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -38,9 +38,9 @@ public FamilyIndexLoader(SampleIndexDBAdaptor sampleIndexDBAdaptor, VariantHadoo this.mrExecutor = mrExecutor; } - public DataResult> load(String study, List> trios, ObjectMap options) throws StorageEngineException { + public DataResult load(String study, List trios, ObjectMap options) throws StorageEngineException { trios = new LinkedList<>(trios); - DataResult> dr = new DataResult<>(); + DataResult dr = new DataResult<>(); dr.setResults(trios); dr.setEvents(new LinkedList<>()); @@ -55,35 +55,34 @@ public DataResult> load(String study, List> trios, Obj options.put(FamilyIndexDriver.SAMPLE_INDEX_VERSION, version); options.put(FamilyIndexDriver.OUTPUT, sampleIndexDBAdaptor.getSampleIndexTableName(studyId, version)); - Iterator> iterator = trios.iterator(); + Iterator iterator = trios.iterator(); while (iterator.hasNext()) { - List trioIds = new ArrayList<>(3); - List trio = iterator.next(); - for (String sample : trio) { - Integer sampleId; - if (sample.equals("-")) { - sampleId = -1; - } else { - sampleId = metadataManager.getSampleId(studyId, sample); - if (sampleId == null) { - throw new IllegalArgumentException("Sample '" + sample + "' not found."); - } - } - trioIds.add(sampleId); + Trio trio = iterator.next(); + + final Integer fatherId; + final Integer motherId; + final Integer childId; + + childId = metadataManager.getSampleId(studyId, trio.getChild()); + if (trio.getFather() == null) { + fatherId = -1; + } else { + fatherId = metadataManager.getSampleIdOrFail(studyId, trio.getFather()); } - if (trioIds.size() != 3) { - throw new IllegalArgumentException("Found trio with " + trioIds.size() + " members, instead of 3: " + trioIds); + if (trio.getMother() == null) { + motherId = -1; + } else { + motherId = metadataManager.getSampleIdOrFail(studyId, trio.getMother()); } - SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, trioIds.get(2)); + + SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyId, childId); if (!overwrite && sampleMetadata.getFamilyIndexStatus(version) == TaskMetadata.Status.READY) { String msg = "Skip sample " + sampleMetadata.getName() + ". Already precomputed!"; logger.info(msg); dr.getEvents().add(new Event(Event.Type.INFO, msg)); iterator.remove(); } else { - Integer fatherId = trioIds.get(0); boolean fatherDefined = fatherId != -1; - Integer motherId = trioIds.get(1); boolean motherDefined = motherId != -1; if (fatherDefined && !fatherId.equals(sampleMetadata.getFather()) || motherDefined && !motherId.equals(sampleMetadata.getMother())) { @@ -106,14 +105,14 @@ public DataResult> load(String study, List> trios, Obj int batchSize = options.getInt(HadoopVariantStorageOptions.SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR.key(), HadoopVariantStorageOptions.SAMPLE_INDEX_FAMILY_MAX_TRIOS_PER_MR.defaultValue()); - List>> batches = BatchUtils.splitBatches(trios, batchSize); + List> batches = BatchUtils.splitBatches(trios, batchSize); if (batches.size() == 1) { runBatch(study, trios, options, studyId); } else { logger.warn("Unable to run family index in one single MapReduce operation."); logger.info("Split in {} jobs of {} samples each.", batches, batches.get(0).size()); for (int i = 0; i < batches.size(); i++) { - List> batch = batches.get(i); + List batch = batches.get(i); logger.info("Running MapReduce {}/{} over {} trios", i + 1, batches, batch.size()); runBatch(study, batch, options, studyId); } @@ -122,12 +121,14 @@ public DataResult> load(String study, List> trios, Obj return dr; } - private void runBatch(String study, List> trios, ObjectMap options, int studyId) throws StorageEngineException { + private void runBatch(String study, List trios, ObjectMap options, int studyId) throws StorageEngineException { if (trios.size() < 500) { - options.put(FamilyIndexDriver.TRIOS, trios.stream().map(trio -> String.join(",", trio)).collect(Collectors.joining(";"))); + options.put(FamilyIndexDriver.TRIOS, trios.stream() + .map(Trio::serialize) + .collect(Collectors.joining(";"))); } else { CohortMetadata cohortMetadata = metadataManager.registerTemporaryCohort(study, "pendingFamilyIndexSamples", - trios.stream().map(t -> t.get(2)).collect(Collectors.toList())); + trios.stream().map(Trio::getChild).collect(Collectors.toList())); options.put(FamilyIndexDriver.TRIOS_COHORT, cohortMetadata.getName()); options.put(FamilyIndexDriver.TRIOS_COHORT_DELETE, true); @@ -137,7 +138,7 @@ private void runBatch(String study, List> trios, ObjectMap options, tableNameGenerator.getArchiveTableName(studyId), tableNameGenerator.getVariantTableName(), studyId, null, options), - "Precompute mendelian errors for " + (trios.size() == 1 ? "trio " + trios.get(0) : trios.size() + " trios")); + "Precompute mendelian errors for " + (trios.size() == 1 ? "trio " + trios.get(0).serialize() : trios.size() + " trios")); } public void postIndex(int studyId, int version) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java index 3731d3ff6f4..2856385c571 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexQueryParser.java @@ -535,8 +535,12 @@ public SampleIndexQuery parse(Query query) { int extendedFilteringRegion = 0; for (String sample : sampleGenotypeQuery.keySet()) { SampleMetadata sampleMetadata = getSampleMetadata(sampleMetadatas, sample, studyId); - extendedFilteringRegion = Math.max(extendedFilteringRegion, sampleMetadata.getAttributes() - .getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + if (sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)) { + extendedFilteringRegion = extendedFilteringRegionDefault; + } else { + extendedFilteringRegion = Math.max(extendedFilteringRegion, sampleMetadata.getAttributes() + .getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + } } if (extendedFilteringRegion <= 0) { extendedFilteringRegion = extendedFilteringRegionDefault; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java index d050421a160..273ad40ae0d 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexSchema.java @@ -118,6 +118,12 @@ public final class SampleIndexSchema { static final String ANNOTATION_CLINICAL_PREFIX = META_PREFIX + "CL_"; static final byte[] ANNOTATION_CLINICAL_PREFIX_BYTES = Bytes.toBytes(ANNOTATION_CLINICAL_PREFIX); + /** + * The largestVariantLength might be unknown if the sample was loaded before this field was added. + * In that case, a default value for the largestVariantLength will be used. + * See HadoopVariantStorageOptions.SAMPLE_INDEX_QUERY_EXTENDED_REGION_FILTER + */ + public static final String UNKNOWN_LARGEST_VARIANT_LENGTH = "unknownLargestVariantLength"; public static final String LARGEST_VARIANT_LENGTH = "largestVariantLength"; private final int version; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java index 8285823ef2e..aea720d356a 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineBNDTest.java @@ -19,7 +19,7 @@ public class HadoopVariantStorageEngineBNDTest extends VariantStorageEngineBNDTe @Override protected void loadFiles() throws Exception { super.loadFiles(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java index f17dedc0b8f..0082e869319 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineDuplicatedTest.java @@ -15,6 +15,6 @@ public class HadoopVariantStorageEngineDuplicatedTest extends VariantStorageEngi public void testDuplicatedVariant() throws Exception { super.testDuplicatedVariant(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java index 5d77e7dc375..f5ebbbe8fd6 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java @@ -14,6 +14,7 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.testclassification.duration.LongTests; +import org.opencb.opencga.storage.core.StoragePipelineResult; import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.exceptions.StoragePipelineException; import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; @@ -31,6 +32,7 @@ import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexEntry; +import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; import java.net.URI; import java.nio.file.Paths; @@ -134,16 +136,33 @@ public void testMultiChromosomeSplitDataVirtualFile() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_VIRTUAL_FILE.key(), "virtual-variant-test-file.vcf"); - variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr20.variant-test-file.vcf.gz")), - outputUri, true, true, true); + int studyId = variantStorageEngine.getMetadataManager().createStudy(STUDY_NAME).getId(); + int sampleIdMock = variantStorageEngine.getMetadataManager().registerSamples(studyId, Collections.singletonList("NA19660")).get(0); + // Mark one random sample as having unknown largest variant size + // Ensure that the largest variant size is not updated + variantStorageEngine.getMetadataManager().updateSampleMetadata(studyId, sampleIdMock, sampleMetadata -> { + sampleMetadata.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, true); + }); + + StoragePipelineResult result = variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr20.variant-test-file.vcf.gz")), + outputUri, true, true, true).get(0); + + // All samples expected to be updated + assertEquals(4, result.getLoadStats().getInt("updatedSampleMetadata")); VariantStorageMetadataManager mm = variantStorageEngine.getMetadataManager(); - int studyId = mm.getStudyId(STUDY_NAME); for (String sample : SAMPLES) { SampleMetadata sampleMetadata = mm.getSampleMetadata(studyId, mm.getSampleId(studyId, sample)); assertEquals(TaskMetadata.Status.READY, sampleMetadata.getIndexStatus()); assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getAnnotationStatus()); assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getSampleIndexAnnotationStatus(1)); + if (sampleIdMock == sampleMetadata.getId()) { + assertFalse(sample, sampleMetadata.getAttributes().containsKey(SampleIndexSchema.LARGEST_VARIANT_LENGTH)); + assertTrue(sample, sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)); + } else { + assertNotEquals(sample, -1, sampleMetadata.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + assertFalse(sample, sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)); + } } variantStorageEngine.annotate(outputUri, new QueryOptions()); @@ -157,8 +176,11 @@ public void testMultiChromosomeSplitDataVirtualFile() throws Exception { variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_VIRTUAL_FILE.key(), "virtual-variant-test-file.vcf"); - variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), - outputUri, true, true, true); + result = variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr21.variant-test-file.vcf.gz")), + outputUri, true, true, true).get(0); + + // No sample expected to be updated + assertEquals(0, result.getLoadStats().getInt("updatedSampleMetadata")); for (String sample : SAMPLES) { SampleMetadata sampleMetadata = mm.getSampleMetadata(studyId, mm.getSampleId(studyId, sample)); @@ -174,12 +196,29 @@ public void testMultiChromosomeSplitDataVirtualFile() throws Exception { assertEquals(TaskMetadata.Status.READY, sampleMetadata.getSampleIndexAnnotationStatus(1)); } + // Revert the unknown largest variant size + // Ensure that the largest variant size is now updated + variantStorageEngine.getMetadataManager().updateSampleMetadata(studyId, sampleIdMock, sampleMetadata -> { + sampleMetadata.getAttributes().put(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH, false); + }); variantStorageEngine.getOptions().put(VariantStorageOptions.STUDY.key(), STUDY_NAME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_SPLIT_DATA.key(), VariantStorageEngine.SplitData.CHROMOSOME); variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_VIRTUAL_FILE.key(), "virtual-variant-test-file.vcf"); - variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr22.variant-test-file.vcf.gz")), - outputUri); + result = variantStorageEngine.index(Collections.singletonList(getResourceUri("by_chr/chr22.variant-test-file.vcf.gz")), + outputUri).get(0); + // One sample (sampleIdMock) expected to be updated + assertEquals(1, result.getLoadStats().getInt("updatedSampleMetadata")); + + for (String sample : SAMPLES) { + SampleMetadata sampleMetadata = mm.getSampleMetadata(studyId, mm.getSampleId(studyId, sample)); + assertEquals(TaskMetadata.Status.READY, sampleMetadata.getIndexStatus()); + assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getAnnotationStatus()); + assertEquals(TaskMetadata.Status.NONE, sampleMetadata.getSampleIndexAnnotationStatus(1)); + + assertNotEquals(sample, -1, sampleMetadata.getAttributes().getInt(SampleIndexSchema.LARGEST_VARIANT_LENGTH, -1)); + assertFalse(sample, sampleMetadata.getAttributes().getBoolean(SampleIndexSchema.UNKNOWN_LARGEST_VARIANT_LENGTH)); + } for (Variant variant : variantStorageEngine.iterable(new Query(VariantQueryParam.INCLUDE_SAMPLE.key(), ParamConstants.ALL), null)) { String expectedFile = "virtual-variant-test-file.vcf"; diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java index 673fe944ba7..e9e99cdb884 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantHadoopMultiSampleTest.java @@ -384,7 +384,7 @@ public void testMultipleFilesConcurrent(ObjectMap extraParams) throws Exception } studyMetadata = dbAdaptor.getMetadataManager().getStudyMetadata(studyMetadata.getId()); - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); // checkLoadedVariants(expectedVariants, dbAdaptor, PLATINUM_SKIP_VARIANTS); @@ -406,7 +406,7 @@ public void testMultipleFilesConcurrent(ObjectMap extraParams) throws Exception assertNotNull(variantFileMetadata); } - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); } @Test @@ -708,7 +708,7 @@ public StudyMetadata testPlatinumFilesOneByOne(ObjectMap otherParams, int maxFil } } - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); for (Variant variant : dbAdaptor) { System.out.println("variant = " + variant); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java index 884f45f39ff..be47dd08890 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/adaptors/HadoopVariantDBAdaptorMultiFileTest.java @@ -43,11 +43,11 @@ public void before() throws Exception { boolean wasLoaded = loaded; super.before(); if (loaded && !wasLoaded) { - VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri(getClass().getSimpleName())); + VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri()); // for (String study : variantStorageEngine.getDBAdaptor().getStudyConfigurationManager().getStudies(null).keySet()) { // variantStorageEngine.fillMissing(study, new ObjectMap(), false); // } -// VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri(getClass().getSimpleName())); +// VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri()); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java index b083abf3488..0052e805798 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/analysis/gwas/FisherTestDriverTest.java @@ -39,7 +39,7 @@ public class FisherTestDriverTest extends VariantStorageBaseTest implements Hado @Before public void setUp() throws Exception { - localOut = newOutputUri(getClass().getSimpleName()); + localOut = newOutputUri(); } @After diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java index 18692c8f77a..3d9db73719e 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTaskTest.java @@ -159,18 +159,31 @@ public void testOverlapsWith() { @Test public void testIsRegionAfterVariantStart() { - assertTrue(FillGapsTask.isRegionAfterVariantStart(100, 100, new Variant("1:100:-:T"))); - assertFalse(FillGapsTask.isRegionAfterVariantStart(100, 100, new Variant("1:100:A:T"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(101, 101, new Variant("1:100:A:T"))); - - assertFalse(FillGapsTask.isRegionAfterVariantStart(99, 99, new Variant("1:100:AAA:GGG"))); - assertFalse(FillGapsTask.isRegionAfterVariantStart(100, 100, new Variant("1:100:AAA:GGG"))); - assertFalse(FillGapsTask.isRegionAfterVariantStart(101, 100, new Variant("1:100:AAA:GGG"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(101, 101, new Variant("1:100:AAA:GGG"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(102, 102, new Variant("1:100:AAA:GGG"))); - assertTrue(FillGapsTask.isRegionAfterVariantStart(103, 103, new Variant("1:100:AAA:GGG"))); + isRegionAfterVariantStart(100, 100, "1:100:-:T", true, false); + isRegionAfterVariantStart(101, 101, "1:100:A:T", true, false); + + isRegionAfterVariantStart(99, 99, "1:100:AAA:GGG", false, false); + isRegionAfterVariantStart(100, 100, "1:100:AAA:GGG", false, true); + isRegionAfterVariantStart(101, 100, "1:100:AAA:GGG", true, true); + isRegionAfterVariantStart(101, 101, "1:100:AAA:GGG", true, true); + isRegionAfterVariantStart(102, 102, "1:100:AAA:GGG", true, true); + isRegionAfterVariantStart(103, 103, "1:100:AAA:GGG", true, false); + + isRegionAfterVariantStart(100, 100, "1:101:-:TTT", false, false); + isRegionAfterVariantStart(101, 100, "1:100:A:T", true, false); + + // Overlapping positions + isRegionAfterVariantStart(100, 110, "1:105:A:T", true, true); + isRegionAfterVariantStart(100, 100, "1:100:A:T", false, true); } + private void isRegionAfterVariantStart(int start, int end, String variantString, boolean afterVariant, boolean overlaps) { + Variant variant = new Variant(variantString); + assertEquals("isRegionAfterVariantStart", afterVariant, FillGapsTask.isRegionAfterVariantStart(start, end, variant)); + assertEquals("overlapsWith", overlaps, FillGapsTask.overlapsWith(variant, variant.getChromosome(), start, end)); + } + + @Test public void fillGapsAlreadyPresent() { FillGapsTask task = new FillGapsTask(metadataManager, this.studyMetadata, true, false, GenotypeClass.UNKNOWN_GENOTYPE); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java index b23ca96b304..3ae2a36bfbc 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/gaps/FillGapsTest.java @@ -1,6 +1,8 @@ package org.opencb.opencga.storage.hadoop.variant.gaps; import com.google.common.collect.BiMap; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -26,10 +28,13 @@ import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor; import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery; +import org.opencb.opencga.storage.hadoop.variant.GenomeHelper; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageOptions; import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest; import org.opencb.opencga.storage.hadoop.variant.adaptors.VariantHadoopDBAdaptor; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixKeyFactory; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexDBAdaptor; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; @@ -79,7 +84,7 @@ public void testFillGapsVcfFiles() throws Exception { inputFiles.add(getResourceUri(fileName)); } - StudyMetadata studyMetadata = load(new ObjectMap(VariantStorageOptions.GVCF.key(), false), inputFiles, newOutputUri(1)); + StudyMetadata studyMetadata = load(new ObjectMap(VariantStorageOptions.GVCF.key(), false), inputFiles, newOutputUri()); // Do not check new missing genotypes, as the input files does not have missing genotype regions testFillGapsPlatinumFiles(studyMetadata, false); @@ -217,8 +222,10 @@ public void checkConflictingFiles(StudyMetadata studyMetadata) throws Exception sampleIds.sort(Integer::compareTo); List samples = sampleIds.stream().map(samplesMap::get).collect(Collectors.toList()); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); + fillGaps(variantStorageEngine, studyMetadata, samples); - printVariants(studyMetadata, dbAdaptor, newOutputUri(1)); + printVariants(studyMetadata, dbAdaptor, newOutputUri()); checkFillGaps(studyMetadata, dbAdaptor, samples, Collections.singleton("1:10020:A:T")); checkSampleIndexTable(dbAdaptor); @@ -317,6 +324,62 @@ public void testFillGapsCorpasome() throws Exception { .setSamples(Arrays.asList("ISDBM322015", "ISDBM322016", "ISDBM322017", "ISDBM322018")), new ObjectMap()); printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); + load(new ObjectMap(VariantStorageOptions.STUDY.key(), STUDY_NAME_2), + Collections.singletonList(getResourceUri("quartet.variants.annotated.partial.vcf.gz"))); + + checkInputValuesAreUnmodified(STUDY_NAME, STUDY_NAME_2); + } + + @Test + public void testFillGapsGiabChinesse() throws Exception { + ObjectMap options = new ObjectMap(VariantStorageOptions.GVCF.key(), true) + .append(HadoopVariantStorageOptions.ARCHIVE_CHUNK_SIZE.key(), 1000000) + .append(VariantStorageOptions.STUDY.key(), STUDY_NAME); + loadGIAB(options); + printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); + + variantStorageEngine.aggregateFamily(STUDY_NAME, new VariantAggregateFamilyParams() + .setSamples(Arrays.asList("HG005", "HG006", "HG007")), new ObjectMap()); + printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); + + options.put(VariantStorageOptions.STUDY.key(), STUDY_NAME_2); + loadGIAB(options); + + checkInputValuesAreUnmodified(STUDY_NAME, STUDY_NAME_2); + } + + private void checkInputValuesAreUnmodified(String aggregatedStudy, String referenceStudy) throws Exception { + VariantHadoopDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); + + int studyId1 = dbAdaptor.getMetadataManager().getStudyId(aggregatedStudy); + int studyId2 = dbAdaptor.getMetadataManager().getStudyId(referenceStudy); + + dbAdaptor.getHBaseManager().act(dbAdaptor.getVariantTable(), table -> { + table.getScanner(new Scan()).iterator().forEachRemaining(r -> { + byte[] row = r.getRow(); + Variant variant = VariantPhoenixKeyFactory.extractVariantFromVariantRowKey(row); + + NavigableMap cells = r.getFamilyMap(GenomeHelper.COLUMN_FAMILY_BYTES); + for (Map.Entry entry : cells.entrySet()) { + String columnKey = Bytes.toString(entry.getKey()); + Integer studyId = VariantPhoenixSchema.extractStudyId(columnKey, false); + if (studyId != null && studyId == studyId2) { + String otherColumnKey = columnKey.replaceFirst(VariantPhoenixSchema.buildStudyColumnsPrefix(studyId2), + VariantPhoenixSchema.buildStudyColumnsPrefix(studyId1)); + byte[] thisCell = entry.getValue(); + byte[] otherCell = cells.get(Bytes.toBytes(otherColumnKey)); + assertArrayEquals(variant.toString() + " study1ColumnKey " + otherColumnKey + ", study2ColumnKey " + columnKey, + thisCell, otherCell); + } + } + }); + }); + } + + private void loadGIAB(ObjectMap options) throws Exception { + load(options, Collections.singletonList(getResourceUri("giab_chinesse/HG005_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz")), newOutputUri(), false); + load(options, Collections.singletonList(getResourceUri("giab_chinesse/HG006_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz")), newOutputUri(), false); + load(options, Collections.singletonList(getResourceUri("giab_chinesse/HG007_GRCh38_1_22_v4.2.1_benchmark.partial.vcf.gz")), newOutputUri(), false); } @Test @@ -325,12 +388,12 @@ public void testFillGapsImpact() throws Exception { ObjectMap extraParams = new ObjectMap(VariantStorageOptions.LOAD_HOM_REF.key(), true); // extraParams.append(VariantStorageOptions.TRANSFORM_FORMAT.key(), "proto"); // extraParams.append(VariantStorageOptions.GVCF.key(), true); - StudyMetadata study = load(extraParams, Collections.singletonList(getResourceUri("impact/HG005_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); + load(extraParams, Collections.singletonList(getResourceUri("impact/HG005_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); load(extraParams, Collections.singletonList(getResourceUri("impact/HG006_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); load(extraParams, Collections.singletonList(getResourceUri("impact/HG007_GRCh38_1_22_v4.2.1_benchmark.tuned.chr6-31.vcf.gz")), uri, false); printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); - variantStorageEngine.aggregateFamily(study.getName(), new VariantAggregateFamilyParams() + variantStorageEngine.aggregateFamily(STUDY_NAME, new VariantAggregateFamilyParams() .setSamples(Arrays.asList("HG005", "HG006", "HG007")), new ObjectMap()); printVariants((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor(), newOutputUri()); @@ -373,11 +436,11 @@ private StudyMetadata loadPlatinum(ObjectMap extraParams, int from, int to) thro inputFiles.add(getResourceUri(fileName)); } - return load(extraParams, inputFiles, newOutputUri(1)); + return load(extraParams, inputFiles, newOutputUri()); } private StudyMetadata load(ObjectMap extraParams, List inputFiles) throws Exception { - return load(extraParams, inputFiles, newOutputUri(1)); + return load(extraParams, inputFiles, newOutputUri()); } private StudyMetadata load(ObjectMap extraParams, List inputFiles, URI outputUri) throws Exception { diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java index 630ddb4960d..034e676c1b4 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/family/FamilyIndexTest.java @@ -18,6 +18,7 @@ import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams; import org.opencb.opencga.core.response.VariantQueryResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; import org.opencb.opencga.storage.core.variant.adaptors.GenotypeClass; @@ -80,12 +81,13 @@ public void before() throws Exception { runETL(variantStorageEngine, getResourceUri("variant-test-me.vcf"), outputUri, params, true, true, true); - List family = Arrays.asList(father, mother, child); + Trio family = new Trio(father, mother, child); + Trio family2 = new Trio("FATHER", "MOTHER", "PROBAND"); - variantStorageEngine.aggregateFamily(study, new VariantAggregateFamilyParams(family, false), new ObjectMap()); + variantStorageEngine.aggregateFamily(study, new VariantAggregateFamilyParams(family.toList(), false), new ObjectMap()); variantStorageEngine.familyIndex(study, Collections.singletonList(family), new ObjectMap()); - variantStorageEngine.familyIndex(study, Collections.singletonList(Arrays.asList("FATHER", "MOTHER", "PROBAND")), new ObjectMap()); + variantStorageEngine.familyIndex(study, Collections.singletonList(family2), new ObjectMap()); variantStorageEngine.annotate(outputUri, new ObjectMap()); diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java index 0319c2bc43b..3e6ab96edcd 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/index/sample/SampleIndexTest.java @@ -31,6 +31,7 @@ import org.opencb.opencga.storage.core.exceptions.StorageEngineException; import org.opencb.opencga.storage.core.metadata.models.SampleMetadata; import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; +import org.opencb.opencga.storage.core.metadata.models.Trio; import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest; import org.opencb.opencga.storage.core.variant.VariantStorageEngine; import org.opencb.opencga.storage.core.variant.VariantStorageOptions; @@ -111,12 +112,12 @@ public class SampleIndexTest extends VariantStorageBaseTest implements HadoopVar // Arrays.asList("NA19661", "NA19685", "NA19600"), // Arrays.asList("NA19685", "NA19600", "NA19660") // ); - private static List> trios = Arrays.asList( - Arrays.asList("NA19660", "NA19661", "NA19685"), - Arrays.asList("NA19660", "NA19661", "NA19600") + private static List trios = Arrays.asList( + new Trio("NA19660", "NA19661", "NA19685"), + new Trio("NA19660", "NA19661", "NA19600") ); - private static List> triosPlatinum = Arrays.asList( - Arrays.asList("NA12877", "-", "NA12878") + private static List triosPlatinum = Arrays.asList( + new Trio("NA12877", null, "NA12878") ); @Before @@ -341,8 +342,8 @@ public void regenerateSampleIndex() throws Exception { studyId, Collections.emptySet(), options), ""); - if (sampleNames.get(study).containsAll(trios.get(0))) { - options.put(FamilyIndexDriver.TRIOS, trios.stream().map(trio -> String.join(",", trio)).collect(Collectors.joining(";"))); + if (sampleNames.get(study).containsAll(trios.get(0).toList())) { + options.put(FamilyIndexDriver.TRIOS, trios.stream().map(Trio::serialize).collect(Collectors.joining(";"))); options.put(FamilyIndexDriver.OVERWRITE, true); new TestMRExecutor().run(FamilyIndexDriver.class, FamilyIndexDriver.buildArgs( dbAdaptor.getArchiveTableName(studyId), @@ -350,7 +351,7 @@ public void regenerateSampleIndex() throws Exception { studyId, Collections.emptySet(), options), ""); } else if (study.equals(STUDY_NAME_3)) { - options.put(FamilyIndexDriver.TRIOS, triosPlatinum.stream().map(trio -> String.join(",", trio)).collect(Collectors.joining(";"))); + options.put(FamilyIndexDriver.TRIOS, triosPlatinum.stream().map(Trio::serialize).collect(Collectors.joining(";"))); options.put(FamilyIndexDriver.OVERWRITE, true); new TestMRExecutor().run(FamilyIndexDriver.class, FamilyIndexDriver.buildArgs( dbAdaptor.getArchiveTableName(studyId), @@ -1036,8 +1037,8 @@ public void testApproximateCount() { @Test public void testFamilyIndexQueryCount() { - List trio = trios.get(0); - String proband = trio.get(2); + Trio trio = trios.get(0); + String proband = trio.getChild(); VariantQueryResult result = variantStorageEngine.get( new Query() .append(STUDY.key(), STUDY_NAME) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java index d0c7fcfc95c..ddb7e993549 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java @@ -235,7 +235,7 @@ public void exportFromSearchIndex() throws Exception { public URI getOutputUri(String fileName) throws IOException { if (exportToLocal) { - return newOutputUri(1).resolve(fileName); + return newOutputUri().resolve(fileName); } else { return URI.create("hdfs:///" + fileName); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java index d63e4c9b6cc..3cce6445d52 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerMultiFilesTest.java @@ -51,7 +51,7 @@ public void tearDown() throws Exception { public void calculateStatsMultiCohortsAfterFillMissingTest() throws Exception { VariantStorageEngine storageEngine = getVariantStorageEngine(); storageEngine.aggregate(studyMetadata.getName(), new VariantAggregateParams(false, false), new ObjectMap()); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); calculateStatsMultiCohortsTest(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java index abebe1e78ab..c98505eeafa 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/HadoopVariantStatisticsManagerTest.java @@ -82,14 +82,14 @@ public void tearDown() throws Exception { public void calculateStatsMultiCohortsTest() throws Exception { super.calculateStatsMultiCohortsTest(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); //delete all stats List cohorts = new ArrayList<>(); metadataManager.getCalculatedCohorts(studyMetadata.getId()).forEach(c -> cohorts.add(c.getName())); variantStorageEngine.deleteStats(studyMetadata.getName(), cohorts, new ObjectMap()); checkCohorts(dbAdaptor, studyMetadata); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); super.calculateStatsMultiCohortsTest(); } @@ -98,14 +98,14 @@ public void calculateStatsMultiCohortsTest() throws Exception { public void calculateStatsSeparatedCohortsTest() throws Exception { super.calculateStatsSeparatedCohortsTest(); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); //delete all stats List cohorts = new ArrayList<>(); metadataManager.getCalculatedCohorts(studyMetadata.getId()).forEach(c -> cohorts.add(c.getName())); variantStorageEngine.deleteStats(studyMetadata.getName(), cohorts, new ObjectMap()); checkCohorts(dbAdaptor, studyMetadata); - VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri()); super.calculateStatsSeparatedCohortsTest(); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java index 1fa042eed74..5a3a0a1ea54 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/stats/SampleVariantStatsTest.java @@ -69,7 +69,7 @@ public void before() throws Exception { engine.annotate(outputUri, new ObjectMap()); engine.aggregateFamily(study, new VariantAggregateFamilyParams(family, false), new ObjectMap()); - VariantHbaseTestUtils.printVariants(engine.getDBAdaptor(), newOutputUri(getTestName().getMethodName())); + VariantHbaseTestUtils.printVariants(engine.getDBAdaptor(), newOutputUri()); stats = computeSampleVariantStatsDirectly(); }