Merge pull request #202 from expani/issue-11415-2

Fixed tests depending on Lucene 7.x Index to start using Lucene 8.x index ( OS 1.3 )

server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectory.java

@@ -38,7 +38,7 @@
  */
 public final class RemoteSnapshotDirectory extends Directory {
 
-    public static final Version SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION = LegacyESVersion.V_6_0_0;
+    public static final Version SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION = LegacyESVersion.V_7_2_0;
 
     private static final String VIRTUAL_FILE_PREFIX = BlobStoreRepository.VIRTUAL_DATA_BLOB_PREFIX;
 

server/src/test/java/org/opensearch/common/lucene/LuceneTests.java

@@ -107,6 +107,8 @@
 public class LuceneTests extends OpenSearchTestCase {
     private static final NamedWriteableRegistry EMPTY_REGISTRY = new NamedWriteableRegistry(Collections.emptyList());
 
+    public static final String OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH = "/indices/bwc/os-1.3.0/testIndex-os-1.3.0.zip";
+
     public void testCleanIndex() throws IOException {
         MockDirectoryWrapper dir = newMockDirectory();
         IndexWriterConfig iwc = newIndexWriterConfig();
@@ -329,13 +331,12 @@ public void testNumDocs() throws IOException {
 
     /**
      * Tests whether old segments are readable and queryable based on the data documented
-     * in the README <a href="file:../../../../../resources/indices/bwc/es-6.3.0/README.md">here</a>.
+     * in the README <a href="file:../../../../../resources/indices/bwc/os-1.3.0/README.md">here</a>.
      */
     public void testReadSegmentInfosExtendedCompatibility() throws IOException {
-        final String pathToTestIndex = "/indices/bwc/es-6.3.0/testIndex-es-6.3.0.zip";
-        final Version minVersion = LegacyESVersion.V_6_0_0;
+        final Version minVersion = LegacyESVersion.V_7_2_0;
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(pathToTestIndex), tmp);
+        TestUtil.unzip(getClass().getResourceAsStream(OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
         try (MockDirectoryWrapper dir = newMockFSDirectory(tmp)) {
             // The standard API will throw an exception
             expectThrows(IndexFormatTooOldException.class, () -> Lucene.readSegmentInfos(dir));

server/src/test/java/org/opensearch/index/engine/ReadOnlyEngineTests.java

@@ -38,6 +38,7 @@
 import org.apache.lucene.tests.util.TestUtil;
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.lucene.LuceneTests;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
@@ -243,9 +244,8 @@ public void testReadOldIndices() throws Exception {
         IOUtils.close(engine, store);
         // The index has one document in it, so the checkpoint cannot be NO_OPS_PERFORMED
         final AtomicLong globalCheckpoint = new AtomicLong(0);
-        final String pathToTestIndex = "/indices/bwc/es-6.3.0/testIndex-es-6.3.0.zip";
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(pathToTestIndex), tmp);
+        TestUtil.unzip(getClass().getResourceAsStream(LuceneTests.OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
         FeatureFlagSetter.set(FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY);
         final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
             "index",
@@ -266,9 +266,8 @@ public void testReadOldIndicesFailure() throws IOException {
         IOUtils.close(engine, store);
         // The index has one document in it, so the checkpoint cannot be NO_OPS_PERFORMED
         final AtomicLong globalCheckpoint = new AtomicLong(0);
-        final String pathToTestIndex = "/indices/bwc/es-6.3.0/testIndex-es-6.3.0.zip";
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(pathToTestIndex), tmp);
+        TestUtil.unzip(getClass().getResourceAsStream(LuceneTests.OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
         try (Store store = createStore(newFSDirectory(tmp))) {
             EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
             try {

server/src/test/java/org/opensearch/index/store/StoreTests.java

@@ -69,6 +69,7 @@
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.lucene.Lucene;
+import org.opensearch.common.lucene.LuceneTests;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
@@ -1284,9 +1285,8 @@ public void testSegmentReplicationDiff() {
     @SuppressForbidden(reason = "sets the SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY feature flag")
     public void testReadSegmentsFromOldIndices() throws Exception {
         int expectedIndexCreatedVersionMajor = SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION.luceneVersion.major;
-        final String pathToTestIndex = "/indices/bwc/es-6.3.0/testIndex-es-6.3.0.zip";
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(pathToTestIndex), tmp);
+        TestUtil.unzip(getClass().getResourceAsStream(LuceneTests.OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
         final ShardId shardId = new ShardId("index", "_na_", 1);
         Store store = null;
 
@@ -1309,10 +1309,9 @@ public void testReadSegmentsFromOldIndices() throws Exception {
     }
 
     public void testReadSegmentsFromOldIndicesFailure() throws IOException {
-        final String pathToTestIndex = "/indices/bwc/es-6.3.0/testIndex-es-6.3.0.zip";
         final ShardId shardId = new ShardId("index", "_na_", 1);
         Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(pathToTestIndex), tmp);
+        TestUtil.unzip(getClass().getResourceAsStream(LuceneTests.OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
             "index",
             Settings.builder()

server/src/test/resources/indices/bwc/es-6.3.0/README.md → server/src/test/resources/indices/bwc/os-1.3.0/README.md

@@ -1,6 +1,6 @@
-# README for _testIndex-es-6.3.0.zip_
+# README for _testIndex-os-1.3.0.zip_
 
-This zip file holds a Lucene index created using ElasticSearch 6.3.0.
+This zip file holds a Lucene index created using OpenSearch 1.3.0 downloaded from https://opensearch.org/versions/opensearch-1-3-0.html
 It was created by running the underlying commands against a single-node cluster,
 then compressing the contents of the underlying Lucene index directory i.e.
 the files under `<elasticsearch-root>/data/nodes/0/indices/<index-uuid>/0/index`.
@@ -16,7 +16,6 @@ curl -X PUT -H 'Content-Type: application/json' 'localhost:9200/testindex?pretty
     "number_of_replicas": 0
   },
   "mappings": {
-    "testData": {
       "properties": {
         "id": { "type": "keyword" },
         "isTestData": { "type": "boolean" },
@@ -35,11 +34,10 @@ curl -X PUT -H 'Content-Type: application/json' 'localhost:9200/testindex?pretty
         "testDate": { "type": "date" },
         "testLocation": { "type": "geo_point"}
       }
-    }
   }
 }'
 
-curl -X POST "localhost:9200/testindex/testData/?pretty" -H 'Content-Type: application/json' -d'
+curl -X POST "localhost:9200/testindex/_doc/1" -H 'Content-Type: application/json' -d'
 {
   "id": "testData1",
   "isTestData": true,
@@ -54,4 +52,7 @@ curl -X POST "localhost:9200/testindex/testData/?pretty" -H 'Content-Type: appli
   "testLocation": "48.553532,-113.022881"
 }
 '
+
+curl -X POST "localhost:9200/testindex/_flush" -H 'Content-Type: application/json' -d'{}'
 ```
+Flushing is essential to commit the changes to ensure tests don't fail complaining there are no docs in segment.