Merge pull request #2481 from opencb/TASK-6266

TASK-6266 - OpenCGA-Storage test failed on develop branch
opencb · Jul 12, 2024 · 6d0b256 · 6d0b256
2 parents 344c4aa + b78eee5
commit 6d0b256
Show file tree

Hide file tree

Showing 14 changed files with 87 additions and 40 deletions.
diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -90,7 +90,7 @@ jobs:
       - name: Maven build
         run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Publish Test Report on GitHub
         uses: scacap/action-surefire-report@v1
         env:

diff --git a/...rc/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java b/...rc/test/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationAnalysisTest.java
@@ -48,6 +48,7 @@
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.testclassification.duration.MediumTests;
 import org.opencb.opencga.core.tools.result.ExecutionResult;
+import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
 
 import java.io.File;
@@ -138,8 +139,11 @@ public void customAnalysisFromClinicalAnalysisTest() throws Exception {
                 .setConfig(config);
 
         ExecutionResult result = customAnalysis.start();
-
-        checkInterpretation(238, result);
+        int expected = opencga.getVariantStorageManager().get(new VariantQuery(query),
+                        new QueryOptions(QueryOptions.LIMIT, 500), clinicalTest.token)
+                .getResults().size();
+        assertNotEquals(0, expected);
+        checkInterpretation(expected, result);
     }
 
     @Test

diff --git a/...ysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/...ysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java
@@ -532,13 +532,20 @@ public void testCellbaseConfigure() throws Exception {
         assertEquals("GRCh38", cellBaseUtils.getAssembly());
 
         String newCellbase = "https://uk.ws.zettagenomics.com/cellbase/";
-        String newCellbaseVersion = "v5.8";
+        String newCellbaseVersion = "v5.2";
+        String newCellbaseDataRelease = "1";
 
         assertNotEquals(newCellbase, cellBaseUtils.getURL());
         assertNotEquals(newCellbaseVersion, cellBaseUtils.getVersion());
+        assertNotEquals(newCellbaseDataRelease, cellBaseUtils.getDataRelease());
 
-        variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, "1", ""), false, null, token);
+        variantStorageManager.setCellbaseConfiguration(project, new CellBaseConfiguration(newCellbase, newCellbaseVersion, newCellbaseDataRelease, ""), false, null, token);
         CellBaseConfiguration cellbaseConfiguration = catalogManager.getProjectManager().get(project, new QueryOptions(), token).first().getCellbase();
+
+        assertEquals(newCellbase, cellbaseConfiguration.getUrl());
+        assertEquals(newCellbaseVersion, cellbaseConfiguration.getVersion());
+        assertEquals(newCellbaseDataRelease, cellbaseConfiguration.getDataRelease());
+
 //        assertTrue(family.getPedigreeGraph() != null);
     }
 

diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java
@@ -220,9 +220,13 @@ private void validateProjectForCreation(String organizationId, Project project)
             throw new CatalogParameterException("Missing mandatory organism information");
         }
         try {
+            //TODO: Should the datarelease be undefined? When undefined, it'd be read from cellbase meta endpoints.
+            String defaultDataRelease = project.getOrganism().getAssembly().equalsIgnoreCase("grch38")
+                    ? ParamConstants.CELLBASE_DATA_RELEASE_GRCH38
+                    : null;
             CellBaseConfiguration cellBaseConfiguration = ParamUtils.defaultObject(project.getCellbase(),
                     new CellBaseConfiguration(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION,
-                            ParamConstants.CELLBASE_DATA_RELEASE, ParamConstants.CELLBASE_APIKEY));
+                            defaultDataRelease, ParamConstants.CELLBASE_APIKEY));
             cellBaseConfiguration = CellBaseValidator.validate(cellBaseConfiguration, project.getOrganism().getScientificName(),
                     project.getOrganism().getAssembly(), true);
             project.setCellbase(cellBaseConfiguration);

diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java
@@ -81,7 +81,7 @@ public class ParamConstants {
 
     public static final String CELLBASE_URL = "https://ws.zettagenomics.com/cellbase";
     public static final String CELLBASE_VERSION = "v5.8";
-    public static final String CELLBASE_DATA_RELEASE = "7";
+    public static final String CELLBASE_DATA_RELEASE_GRCH38 = "7";
     public static final String CELLBASE_APIKEY = "";
 
     public static final String POP_FREQ_1000G_CB_V4 = "1kG_phase3";

diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java b/opencga-core/src/main/java/org/opencb/opencga/core/cellbase/CellBaseValidator.java
@@ -171,7 +171,7 @@ private CellBaseConfiguration validate(boolean autoComplete) throws IOException
             logger.warn("DataRelease not supported on version '" + serverVersion + ".x'");
         } else {
             String dataRelease = getDataRelease();
-            if (dataRelease == null) {
+            if (StringUtils.isEmpty(dataRelease)) {
                 if (autoComplete) {
                     cellBaseConfiguration.setDataRelease(getDefaultDataRelease());
                 } else {

diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/storage/CellBaseConfiguration.java
@@ -46,7 +46,7 @@ public class CellBaseConfiguration {
     private String apiKey;
 
     public CellBaseConfiguration() {
-        this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE,
+        this(ParamConstants.CELLBASE_URL, ParamConstants.CELLBASE_VERSION, ParamConstants.CELLBASE_DATA_RELEASE_GRCH38,
                 ParamConstants.CELLBASE_APIKEY);
     }
 

diff --git a/...a-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java b/...a-master/src/test/java/org/opencb/opencga/master/monitor/executors/LocalExecutorTest.java
@@ -91,9 +91,9 @@ public void testKill() throws Exception {
         for (int i = 0; i < 10; i++) {
             String jobId = "jobId-" + i;
 //            System.out.println("Checking status of job " + jobId);
-            while(!localExecutor.getStatus(jobId).equals("RUNNING")) {
-                Thread.sleep(10);
-            }
+            do {
+                Thread.sleep(100);
+            } while (!localExecutor.getStatus(jobId).equals("RUNNING"));
             assertEquals("RUNNING", localExecutor.getStatus(jobId));
             assertTrue(localExecutor.kill(jobId));
         }

diff --git a/...re/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java b/...re/src/test/java/org/opencb/opencga/storage/core/variant/VariantStorageEngineBNDTest.java
@@ -47,7 +47,7 @@ public abstract class VariantStorageEngineBNDTest extends VariantStorageBaseTest
     public void before() throws Exception {
         variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL);
         variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION);
-        variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE);
+        variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38);
         if (!loaded) {
             clearDB(DB_NAME);
             loadFiles();
@@ -58,7 +58,7 @@ public void before() throws Exception {
     protected void loadFiles() throws Exception {
         variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL);
         variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION);
-        variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE);
+        variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38);
         studyMetadata = new StudyMetadata(1, "s1");
 //        variantStorageEngine.getOptions().append(VariantStorageOptions.ANNOTATOR_CELLBASE_EXCLUDE.key(), "expression,clinical");
         input1 = getResourceUri("variant-test-bnd.vcf");

diff --git a/.../test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java b/.../test/java/org/opencb/opencga/storage/core/variant/VariantStorageSearchIntersectTest.java
@@ -81,7 +81,7 @@ public void before() throws Exception {
         VariantStorageEngine variantStorageEngine = getVariantStorageEngine();
         variantStorageEngine.getConfiguration().getCellbase().setUrl(ParamConstants.CELLBASE_URL);
         variantStorageEngine.getConfiguration().getCellbase().setVersion(ParamConstants.CELLBASE_VERSION);
-        variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE);
+        variantStorageEngine.getConfiguration().getCellbase().setDataRelease(ParamConstants.CELLBASE_DATA_RELEASE_GRCH38);
         dbAdaptor = variantStorageEngine.getDBAdaptor();
         if (!loaded) {
             loadFile();

diff --git a/...ava/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java b/...ava/org/opencb/opencga/storage/core/variant/query/executors/VariantQueryExecutorTest.java
@@ -47,15 +47,31 @@ public abstract class VariantQueryExecutorTest extends VariantStorageBaseTest {
     private DBAdaptorVariantQueryExecutor dbQueryExecutor;
     private List<VariantQueryExecutor> variantQueryExecutors;
 
-    @ClassRule
-    public static VariantSolrExternalResource solr = new VariantSolrExternalResource();
+    public static VariantSolrExternalResource solr = null;
+
+
+    public void initSolr() throws Exception {
+        if (solr != null) {
+            solr = new VariantSolrExternalResource();
+            solr.before();
+        }
+    }
+
+    @AfterClass
+    public static void afterClass() {
+        if (solr != null) {
+            solr.after();
+        }
+    }
 
     @Before
     public void setUp() throws Exception {
-
+        initSolr();
         VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor();
         VariantStorageMetadataManager metadataManager = dbAdaptor.getMetadataManager();
-        solr.configure(variantStorageEngine);
+        if (solr != null) {
+            solr.configure(variantStorageEngine);
+        }
         if (!fileIndexed) {
             studyMetadata = newStudyMetadata();
 //            variantSource = new VariantSource(smallInputUri.getPath(), "testAlias", "testStudy", "Study for testing purposes");
@@ -96,9 +112,11 @@ public void setUp() throws Exception {
             variantStorageEngine.calculateStats(studyMetadata.getName(),
                     new ArrayList<>(cohorts.keySet()), options);
 
-            solr.configure(variantStorageEngine);
-            variantStorageEngine.secondaryIndex();
-            Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive());
+            if (solr != null) {
+                solr.configure(variantStorageEngine);
+                variantStorageEngine.secondaryIndex();
+                Assert.assertTrue(variantStorageEngine.secondaryAnnotationIndexActiveAndAlive());
+            }
 
             variantQueryExecutors = variantStorageEngine.getVariantQueryExecutors();
             dbQueryExecutor = null;
@@ -135,8 +153,8 @@ public void testXRefRs() throws StorageEngineException {
                 with("ConsequenceType", VariantAnnotation::getConsequenceTypes, hasItem(
                         with("GeneName", ConsequenceType::getGeneName,
                                 is("TEX13B"))))));
-        matchers.put("COSV60260399", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem(
-                with("Cosmic", EvidenceEntry::getId, is("COSV60260399"))))));
+        matchers.put("RCV000155534", hasAnnotation(with("TraitAssociation", VariantAnnotation::getTraitAssociation, hasItem(
+                with("Clinvar", EvidenceEntry::getId, is("RCV000155534"))))));
         matchers.put("ENST00000341832.11(ENSG00000248333):c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(
                  is("ENST00000341832.11(ENSG00000248333):c.356-1170A>G")))));
         matchers.put("ENST00000341832.11:c.356-1170A>G", hasAnnotation(with("HGVS", VariantAnnotation::getHgvs, hasItem(

diff --git a/...-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java b/...-core/src/test/java/org/opencb/opencga/storage/core/variant/search/VariantSearchTest.java
@@ -187,15 +187,16 @@ public void testSpecialCharacter() throws Exception {
         List<Variant> variants = getVariants(limit);
         List<Variant> annotatedVariants = annotatedVariants(variants);
 
-        String studyId = "abyu12";
-        String fileId = "a.vcf";
+        String study = "abyu12";
+        String file = "a.vcf";
 
-        variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(fileId);
+        variants.get(0).getStudies().get(0).getFiles().get(0).setFileId(file);
         System.out.println(variants.get(0).getStudies().get(0).getFiles().get(0).getFileId());
         //System.exit(-1);
 
-        scm.createStudy(studyId);
-
+        int studyId = scm.createStudy(study).getId();
+        int fileId = scm.registerFile(studyId, file, Arrays.asList("A-A", "B", "C", "D"));
+        scm.addIndexedFiles(studyId, Collections.singletonList(fileId));
         String collection = solr.coreName;
         variantSearchManager.create(collection);
 
@@ -204,13 +205,13 @@ public void testSpecialCharacter() throws Exception {
         samplePosition.put("B", 1);
         samplePosition.put("C", 2);
         samplePosition.put("D", 3);
-        annotatedVariants.get(0).getStudies().get(0).setStudyId(studyId).setSortedSamplesPosition(samplePosition);
+        annotatedVariants.get(0).getStudies().get(0).setStudyId(study).setSortedSamplesPosition(samplePosition);
         variantSearchManager.insert(collection, annotatedVariants);
 
         Query query = new Query();
-        query.put(VariantQueryParam.STUDY.key(), studyId);
+        query.put(VariantQueryParam.STUDY.key(), study);
 //        query.put(VariantQueryParam.SAMPLE.key(), samplePosition.keySet().toArray()[0]);
-        query.put(VariantQueryParam.FILE.key(), fileId);
+        query.put(VariantQueryParam.FILE.key(), file);
         query.put(VariantQueryParam.FILTER.key(), "PASS");
         query.put(VariantQueryParam.ANNOT_CLINICAL_SIGNIFICANCE.key(), "benign");
         VariantQueryResult<Variant> results = variantSearchManager.query(collection, variantStorageEngine.parseQuery(query,

diff --git a/...va/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java b/...va/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngineSplitDataTest.java
@@ -224,6 +224,10 @@ public void testMultiChromosomeSplitData() throws Exception {
     }
 
     private void failAtLoadingFile(String x, String file1, URI outputUri) throws Exception {
+        failAtLoadingFile(x, file1, outputUri, 1);
+    }
+
+    private void failAtLoadingFile(String x, String file1, URI outputUri, int expectedRunningTasks) throws Exception {
         try {
             VariantStorageEngine engine = getMockedStorageEngine(new ObjectMap(VariantStorageOptions.STUDY.key(), STUDY_NAME));
             engine.index(Collections.singletonList(getResourceUri(x + file1)), outputUri);
@@ -236,11 +240,13 @@ private void failAtLoadingFile(String x, String file1, URI outputUri) throws Exc
                 assertEquals(TaskMetadata.Status.NONE, fileMetadata.getIndexStatus());
                 List<TaskMetadata> runningTasks = new ArrayList<>();
                 metadataManager.getRunningTasks(studyId).forEach(runningTasks::add);
-                assertEquals(1, runningTasks.size());
-                assertEquals(TaskMetadata.Type.LOAD, runningTasks.get(0).getType());
-                assertEquals(TaskMetadata.Status.RUNNING, runningTasks.get(0).currentStatus());
-                assertEquals(Arrays.asList(fileMetadata.getId()), runningTasks.get(0).getFileIds());
+                assertEquals(expectedRunningTasks, runningTasks.size());
+                TaskMetadata taskMetadata = runningTasks.get(runningTasks.size() - 1);
+                assertEquals(TaskMetadata.Type.LOAD, taskMetadata.getType());
+                assertEquals(TaskMetadata.Status.RUNNING, taskMetadata.currentStatus());
+                assertEquals(Arrays.asList(fileMetadata.getId()), taskMetadata.getFileIds());
             } catch (AssertionError error) {
+                error.addSuppressed(e);
                 e.printStackTrace();
                 throw error;
             }
@@ -595,7 +601,7 @@ public void testLoadMultiFileDataConcurrencyDeleteMany() throws Exception {
         String file2 = "1K.end.platinum-genomes-vcf-NA12878_S1.vcf.gz";
 
         failAtLoadingFile(resourceDir, file1, outDir);
-        failAtLoadingFile(resourceDir, file2, outDir);
+        failAtLoadingFile(resourceDir, file2, outDir, 2);
 //        try {
 //            getMockedStorageEngine().index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir);
 //            fail("Should have thrown an exception");
@@ -657,7 +663,7 @@ public void testLoadMultiFileDataConcurrencyFail() throws Exception {
 
         variantStorageEngine.getOptions().put(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), true);
         variantStorageEngine.getOptions().put(VariantStorageOptions.RESUME.key(), true);
-        variantStorageEngine.index(Collections.singletonList(getResourceUri(file1)), outDir);
+        variantStorageEngine.index(Collections.singletonList(getResourceUri(resourceDir + file1)), outDir);
 
     }
 

diff --git a/...opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java b/...opencb/opencga/storage/hadoop/variant/query/executors/HadoopVariantQueryExecutorTest.java
@@ -9,15 +9,22 @@
 import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest;
 import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils;
 
-import static org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils.printVariants;
-
 
 @Category(LongTests.class)
 public class HadoopVariantQueryExecutorTest extends VariantQueryExecutorTest implements HadoopVariantStorageTest {
 
     @ClassRule
     public static ExternalResource externalResource = new HadoopExternalResource();
 
+    @Override
+    public void initSolr() throws Exception {
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+            super.initSolr();
+        } else {
+            System.out.println("Solr testing not available");
+        }
+    }
+
     @Override
     @Before
     public void setUp() throws Exception {