storage: Allow reloading an indexed variant file. #TASK-4794

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java

@@ -100,6 +100,7 @@ public class VariantFileIndexerOperationManager extends OperationManager {
     private List<File> filesToIndex;
     private CatalogStorageMetadataSynchronizer synchronizer;
     private boolean fullSynchronize = false;
+    private boolean force;
 
     public VariantFileIndexerOperationManager(VariantStorageManager variantStorageManager, VariantStorageEngine engine) {
         super(variantStorageManager, engine);
@@ -138,6 +139,7 @@ private void check(String study, ObjectMap params, String token) throws Exceptio
         }
         resume = params.getBoolean(VariantStorageOptions.RESUME.key());
         skipIndexedFiles = params.getBoolean(SKIP_INDEXED_FILES);
+        force = params.getBoolean(VariantStorageOptions.FORCE.key());
 
         // Obtain the type of analysis (transform, load or index)
         step = getType(load, transform);
@@ -589,6 +591,7 @@ private List<File> filterTransformFiles(List<File> fileList, boolean resume) thr
                         break;
                     case VariantIndexStatus.INDEXING:
                     case VariantIndexStatus.TRANSFORMING:
+                    case VariantIndexStatus.LOADING:
                         if (resume) {
                             filteredFiles.add(file);
                         } else {
@@ -603,14 +606,17 @@ private List<File> filterTransformFiles(List<File> fileList, boolean resume) thr
                         }
                         break;
                     case VariantIndexStatus.TRANSFORMED:
-                    case VariantIndexStatus.LOADING:
                     case VariantIndexStatus.READY:
                     default:
                         String msg = "We can only " + step + " VCF files not transformed, the status is " + indexStatus;
-                        if (skipIndexedFiles) {
-                            logger.warn(msg);
+                        if (force) {
+                            filteredFiles.add(file);
                         } else {
-                            throw new StorageEngineException(msg);
+                            if (skipIndexedFiles) {
+                                logger.warn(msg);
+                            } else {
+                                throw new StorageEngineException(msg);
+                            }
                         }
                         break;
                 }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/operations/VariantIndexOperationTool.java

@@ -90,6 +90,7 @@ protected void check() throws Exception {
         params.put(VariantStorageOptions.ANNOTATE.key(), indexParams.isAnnotate());
         params.putIfNotEmpty(VariantStorageOptions.ANNOTATOR.key(), indexParams.getAnnotator());
         params.put(VariantStorageOptions.ANNOTATION_OVERWEITE.key(), indexParams.isOverwriteAnnotations());
+        params.put(VariantStorageOptions.FORCE.key(), indexParams.isForceReload());
         params.put(VariantStorageOptions.RESUME.key(), indexParams.isResume());
         params.put(VariantStorageOptions.NORMALIZATION_SKIP.key(), indexParams.getNormalizationSkip());
         params.putIfNotEmpty(VariantStorageOptions.NORMALIZATION_REFERENCE_GENOME.key(), indexParams.getReferenceGenome());

opencga-analysis/src/test/java/org/opencb/opencga/analysis/alignment/AlignmentAnalysisTest.java

@@ -18,6 +18,7 @@
 
 import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
@@ -95,6 +96,7 @@ public AlignmentAnalysisTest(String storageEngine) {
     private CatalogManager catalogManager;
     private VariantStorageManager variantStorageManager;
 
+    @ClassRule
     public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource();
     public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource = new HadoopVariantStorageTest.HadoopExternalResource();
 

opencga-analysis/src/test/java/org/opencb/opencga/analysis/rga/RgaSolrExtenalResource.java

@@ -5,26 +5,29 @@
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
 import org.apache.solr.core.NodeConfig;
-import org.apache.solr.core.SolrResourceLoader;
 import org.junit.rules.ExternalResource;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
 import org.opencb.commons.datastore.solr.SolrManager;
-import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
 import org.opencb.opencga.core.common.GitRepositoryState;
+import org.opencb.opencga.core.common.TimeUtils;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
-import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.*;
+import static org.opencb.opencga.storage.core.variant.VariantStorageBaseTest.getResourceUri;
 
 public class RgaSolrExtenalResource extends ExternalResource {
 
     public String coreName = "opencga_rga_test";
 
     private SolrClient solrClient;
     protected boolean embeded = true;
+    private Class<?> testClass;
 
     public RgaSolrExtenalResource() {
         this(true);
@@ -34,16 +37,23 @@ public RgaSolrExtenalResource(boolean embeded) {
         this.embeded = embeded;
     }
 
+    @Override
+    public Statement apply(Statement base, Description description) {
+        testClass = description.getTestClass();
+        return super.apply(base, description);
+    }
+
     @Override
     protected void before() throws Throwable {
         super.before();
 
-        Path rootDir = getTmpRootDir();
+        Path rootDir = Paths.get("target/test-data", "junit-rga-solr-" + TimeUtils.getTimeMillis());
+        Files.createDirectories(rootDir);
 
         String mainConfigSet = "opencga-rga-configset-" + GitRepositoryState.get().getBuildVersion();
         String auxConfigSet = "opencga-rga-aux-configset-" + GitRepositoryState.get().getBuildVersion();
-        copyConfigSetConfiguration(mainConfigSet, "managed-schema");
-        copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema");
+        copyConfigSetConfiguration(mainConfigSet, "managed-schema", rootDir);
+        copyConfigSetConfiguration(auxConfigSet, "aux-managed-schema", rootDir);
 
         String solrHome = rootDir.resolve("solr").toString();
 
@@ -80,15 +90,15 @@ protected void after() {
         }
     }
 
-    private void copyConfigSetConfiguration(String configSet, String managedSchemaFile) throws IOException {
+    private void copyConfigSetConfiguration(String configSet, String managedSchemaFile, Path rootDir) throws IOException {
         // Copy configuration
-        getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml");
-        getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema");
-        getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json");
-        getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt");
-        getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt");
-        getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt");
-        getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt");
+        getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + configSet + "/solrconfig.xml", rootDir);
+        getResourceUri("rga/" + managedSchemaFile, "configsets/" + configSet + "/managed-schema", rootDir);
+        getResourceUri("configsets/variantsCollection/params.json", "configsets/" + configSet + "/params.json", rootDir);
+        getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + configSet + "/protwords.txt", rootDir);
+        getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + configSet + "/stopwords.txt", rootDir);
+        getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + configSet + "/synonyms.txt", rootDir);
+        getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + configSet + "/lang/stopwords_en.txt", rootDir);
     }
 
     public RgaEngine configure(StorageConfiguration storageConfiguration) {

opencga-analysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java

@@ -38,6 +38,7 @@
 import org.opencb.opencga.core.common.YesNoAuto;
 import org.opencb.opencga.core.config.storage.SampleIndexConfiguration;
 import org.opencb.opencga.core.config.storage.StorageConfiguration;
+import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.cohort.Cohort;
 import org.opencb.opencga.core.models.cohort.CohortCreateParams;
 import org.opencb.opencga.core.models.common.IndexStatus;
@@ -57,6 +58,7 @@
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.opencb.opencga.core.testclassification.duration.LongTests;
 import org.opencb.opencga.core.tools.result.ExecutionResult;
+import org.opencb.opencga.storage.core.exceptions.StorageEngineException;
 import org.opencb.opencga.storage.core.metadata.models.VariantScoreMetadata;
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantQueryParam;
@@ -75,6 +77,7 @@
 import static org.hamcrest.CoreMatchers.anyOf;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 @RunWith(Parameterized.class)
 @Category(LongTests.class)
@@ -277,6 +280,29 @@ public void setUpCatalogManager() throws Exception {
 
     }
 
+    @Test
+    public void testVariantFileReload() throws Exception {
+
+        try {
+            toolRunner.execute(VariantIndexOperationTool.class, STUDY,
+                    new VariantIndexParams()
+                            .setForceReload(false)
+                            .setFile(file.getId()),
+                    Paths.get(opencga.createTmpOutdir()), "index_reload", token);
+            fail("Should have thrown an exception");
+        } catch (ToolException e) {
+            assertEquals(StorageEngineException.class, e.getCause().getClass());
+            assertEquals("We can only INDEX VCF files not transformed, the status is READY", e.getCause().getMessage());
+        }
+
+        toolRunner.execute(VariantIndexOperationTool.class, STUDY,
+                new VariantIndexParams()
+                        .setForceReload(true)
+                        .setFile(file.getId()),
+                Paths.get(opencga.createTmpOutdir()), "index_reload", token);
+
+    }
+
     @Test
     public void testVariantSecondaryAnnotationIndex() throws Exception {
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/VariantInternalCommandExecutor.java

@@ -380,6 +380,7 @@ private void index() throws ToolException {
                 cliOptions.genericVariantIndexOptions.family,
                 cliOptions.genericVariantIndexOptions.somatic,
                 cliOptions.genericVariantIndexOptions.load,
+                cliOptions.genericVariantIndexOptions.forceReload,
                 cliOptions.genericVariantIndexOptions.loadSplitData,
                 cliOptions.genericVariantIndexOptions.loadMultiFileData,
                 cliOptions.genericVariantIndexOptions.loadSampleIndex,

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java

@@ -1,5 +1,5 @@
 /*
-* Copyright 2015-2023-04-18 OpenCB
+* Copyright 2015-2023-07-28 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java

@@ -1,5 +1,5 @@
 /*
-* Copyright 2015-2023-04-18 OpenCB
+* Copyright 2015-2023-07-28 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/AnalysisVariantCommandExecutor.java

@@ -862,6 +862,7 @@ private RestResponse<Job> runIndex() throws Exception {
             putNestedIfNotNull(beanParams, "family",commandOptions.family, true);
             putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true);
             putNestedIfNotNull(beanParams, "load",commandOptions.load, true);
+            putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true);
             putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true);
             putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true);
             putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true);

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/OperationsVariantStorageCommandExecutor.java

@@ -530,6 +530,7 @@ private RestResponse<Job> indexVariant() throws Exception {
             putNestedIfNotNull(beanParams, "family",commandOptions.family, true);
             putNestedIfNotNull(beanParams, "somatic",commandOptions.somatic, true);
             putNestedIfNotNull(beanParams, "load",commandOptions.load, true);
+            putNestedIfNotNull(beanParams, "forceReload",commandOptions.forceReload, true);
             putNestedIfNotEmpty(beanParams, "loadSplitData",commandOptions.loadSplitData, true);
             putNestedIfNotNull(beanParams, "loadMultiFileData",commandOptions.loadMultiFileData, true);
             putNestedIfNotEmpty(beanParams, "loadSampleIndex",commandOptions.loadSampleIndex, true);
@@ -600,6 +601,7 @@ private RestResponse<Job> launcherVariantIndex() throws Exception {
             putNestedIfNotNull(beanParams, "indexParams.family",commandOptions.indexParamsFamily, true);
             putNestedIfNotNull(beanParams, "indexParams.somatic",commandOptions.indexParamsSomatic, true);
             putNestedIfNotNull(beanParams, "indexParams.load",commandOptions.indexParamsLoad, true);
+            putNestedIfNotNull(beanParams, "indexParams.forceReload",commandOptions.indexParamsForceReload, true);
             putNestedIfNotEmpty(beanParams, "indexParams.loadSplitData",commandOptions.indexParamsLoadSplitData, true);
             putNestedIfNotNull(beanParams, "indexParams.loadMultiFileData",commandOptions.indexParamsLoadMultiFileData, true);
             putNestedIfNotEmpty(beanParams, "indexParams.loadSampleIndex",commandOptions.indexParamsLoadSampleIndex, true);