From 569e90cf5fc31273f2320d0044476a52b8e82687 Mon Sep 17 00:00:00 2001 From: mulugetam Date: Thu, 13 Apr 2023 12:44:14 -0700 Subject: [PATCH] Fix: avoid ZSTD codec from overriding service codec factory. (#7037) * Fix: enable ZSTD codec only if index.codec is set to ZSTD. - addresses https://github.com/opensearch-project/OpenSearch/issues/7012 Signed-off-by: Mulugeta Mammo * Removed custom CodecService and CodecServiceFactory classes. - Removed custom classes for CodecService and CodecServiceFactory. - Also removed PerFieldMappingPostingFormatCodec -- not required. - Added documentation. Signed-off-by: Mulugeta Mammo * Bump zstd-jni version from 1.5.4-1 to 1.5.5-1. - Zstandard version 1.5.5 contains a bug fix for a rare corruption error described here: https://github.com/facebook/zstd/releases/tag/v1.5.5. The zstd-jni version we use here, 1.5.5-1, uses Zstandard v1.5.5. Signed-off-by: Mulugeta Mammo --------- Signed-off-by: Mulugeta Mammo --- CHANGELOG.md | 2 +- sandbox/plugins/custom-codecs/build.gradle | 2 +- .../licenses/zstd-jni-1.5.4-1.jar.sha1 | 1 - .../licenses/zstd-jni-1.5.5-1.jar.sha1 | 1 + .../codec/customcodecs/CustomCodecPlugin.java | 16 +---- .../customcodecs/CustomCodecService.java | 67 ------------------- .../CustomCodecServiceFactory.java | 27 -------- .../Lucene95CustomStoredFieldsFormat.java | 13 ++++ .../PerFieldMappingPostingFormatCodec.java | 25 ------- .../index/codec/customcodecs/ZstdCodec.java | 1 + .../customcodecs/ZstdCompressionMode.java | 2 + .../codec/customcodecs/ZstdNoDictCodec.java | 1 + .../ZstdNoDictCompressionMode.java | 2 + 13 files changed, 23 insertions(+), 137 deletions(-) delete mode 100644 sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.4-1.jar.sha1 create mode 100644 sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.5-1.jar.sha1 delete mode 100644 sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java delete mode 100644 sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java delete mode 100644 sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/PerFieldMappingPostingFormatCodec.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b195f89c68574..1628db40a1e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,7 +114,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `net.minidev:json-smart` from 2.4.7 to 2.4.10 - Bump `org.apache.maven:maven-model` from 3.6.2 to 3.9.1 - Bump `org.codehaus.jettison:jettison` from 1.5.3 to 1.5.4 ([#6878](https://github.com/opensearch-project/OpenSearch/pull/6878)) -- Add `com.github.luben:zstd-jni:1.5.4-1` ([#3577](https://github.com/opensearch-project/OpenSearch/pull/3577)) +- Add `com.github.luben:zstd-jni:1.5.5-1` ([#3577](https://github.com/opensearch-project/OpenSearch/pull/3577)) - Bump: Netty from 4.1.90.Final to 4.1.91.Final , ASM 9.4 to ASM 9.5, ByteBuddy 1.14.2 to 1.14.3 ([#6981](https://github.com/opensearch-project/OpenSearch/pull/6981)) - Bump `com.azure:azure-storage-blob` from 12.15.0 to 12.21.1 diff --git a/sandbox/plugins/custom-codecs/build.gradle b/sandbox/plugins/custom-codecs/build.gradle index bf1bc719b0ae6..43b134c30da0f 100644 --- a/sandbox/plugins/custom-codecs/build.gradle +++ b/sandbox/plugins/custom-codecs/build.gradle @@ -21,7 +21,7 @@ opensearchplugin { } dependencies { - api "com.github.luben:zstd-jni:1.5.4-1" + api "com.github.luben:zstd-jni:1.5.5-1" } yamlRestTest.enabled = false; diff --git a/sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.4-1.jar.sha1 b/sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.4-1.jar.sha1 deleted file mode 100644 index e95377f702a6c..0000000000000 --- a/sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.4-1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -291ccaacc039e41932de877303edb6af98a91c24 diff --git a/sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.5-1.jar.sha1 b/sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.5-1.jar.sha1 new file mode 100644 index 0000000000000..bfb9e565bc6d5 --- /dev/null +++ b/sandbox/plugins/custom-codecs/licenses/zstd-jni-1.5.5-1.jar.sha1 @@ -0,0 +1 @@ +fda1d6278299af27484e1cc3c79a060e41b7ef7e \ No newline at end of file diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 1e0245f3c8c6b..ed2eb962050d4 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -10,10 +10,6 @@ import org.opensearch.plugins.Plugin; import org.opensearch.plugins.EnginePlugin; -import org.opensearch.index.codec.CodecServiceFactory; -import org.opensearch.index.IndexSettings; - -import java.util.Optional; /** * A plugin that implements custom codecs. Supports these codecs: @@ -25,16 +21,6 @@ * @opensearch.internal */ public final class CustomCodecPlugin extends Plugin implements EnginePlugin { - - /** Creates a new instance */ + /** Creates a new instance. */ public CustomCodecPlugin() {} - - /** - * @param indexSettings is the default indexSettings - * @return the engine factory - */ - @Override - public Optional getCustomCodecServiceFactory(final IndexSettings indexSettings) { - return Optional.of(new CustomCodecServiceFactory()); - } } diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java deleted file mode 100644 index 4dd25caa86d94..0000000000000 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.Codec; -import org.opensearch.common.collect.MapBuilder; -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.mapper.MapperService; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Map; - -/** - * CustomCodecService provides ZSTD and ZSTDNODICT compression codecs. - */ -public class CustomCodecService extends CodecService { - private final Map codecs; - - /** - * Creates a new CustomCodecService. - * - * @param mapperService A mapper service. - * @param logger A logger. - */ - public CustomCodecService(MapperService mapperService, Logger logger) { - super(mapperService, logger); - final MapBuilder codecs = MapBuilder.newMapBuilder(); - if (mapperService == null) { - codecs.put(Lucene95CustomCodec.Mode.ZSTD.name(), new ZstdCodec()); - codecs.put(Lucene95CustomCodec.Mode.ZSTDNODICT.name(), new ZstdNoDictCodec()); - } else { - codecs.put( - Lucene95CustomCodec.Mode.ZSTD.name(), - new PerFieldMappingPostingFormatCodec(Lucene95CustomCodec.Mode.ZSTD, mapperService) - ); - codecs.put( - Lucene95CustomCodec.Mode.ZSTDNODICT.name(), - new PerFieldMappingPostingFormatCodec(Lucene95CustomCodec.Mode.ZSTDNODICT, mapperService) - ); - } - this.codecs = codecs.immutableMap(); - } - - @Override - public Codec codec(String name) { - Codec codec = codecs.get(name); - if (codec == null) { - return super.codec(name); - } - return codec; - } - - @Override - public String[] availableCodecs() { - ArrayList ac = new ArrayList(Arrays.asList(super.availableCodecs())); - ac.addAll(codecs.keySet()); - return ac.toArray(new String[0]); - } -} diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java deleted file mode 100644 index 9a1872abfcbd7..0000000000000 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.codec.CodecServiceConfig; -import org.opensearch.index.codec.CodecServiceFactory; - -/** - * A factory for creating new {@link CodecService} instance - */ -public class CustomCodecServiceFactory implements CodecServiceFactory { - - /** Creates a new instance. */ - public CustomCodecServiceFactory() {} - - @Override - public CodecService createCodecService(CodecServiceConfig config) { - return new CustomCodecService(config.getMapperService(), config.getLogger()); - } -} diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java index e0253516b6d0a..2bfec2ef171d4 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java +++ b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java @@ -61,6 +61,13 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compr zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); } + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ @Override public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { String value = si.getAttribute(MODE_KEY); @@ -71,6 +78,12 @@ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, Fiel return impl(mode).fieldsReader(directory, si, fn, context); } + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ @Override public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { String previous = si.putAttribute(MODE_KEY, mode.name()); diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/PerFieldMappingPostingFormatCodec.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/PerFieldMappingPostingFormatCodec.java deleted file mode 100644 index f1c64853bca40..0000000000000 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/PerFieldMappingPostingFormatCodec.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.index.mapper.MapperService; - -/** PerFieldMappingPostingFormatCodec. {@link org.opensearch.index.codec.PerFieldMappingPostingFormatCodec} */ -public class PerFieldMappingPostingFormatCodec extends Lucene95CustomCodec { - - /** - * Creates a new instance. - * - * @param compressionMode The compression mode (ZSTD or ZSTDNODICT). - * @param mapperService The mapper service. - */ - public PerFieldMappingPostingFormatCodec(Lucene95CustomCodec.Mode compressionMode, MapperService mapperService) { - super(compressionMode); - } -} diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java index 086e2461b1f6a..2b09540d8037d 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java +++ b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java @@ -29,6 +29,7 @@ public ZstdCodec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } + /** The name for this codec. */ @Override public String toString() { return getClass().getSimpleName(); diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 795ddf3ab2d17..ceba6ec731e5a 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -46,11 +46,13 @@ protected ZstdCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } + /** Creates a new compressor instance.*/ @Override public Compressor newCompressor() { return new ZstdCompressor(compressionLevel); } + /** Creates a new decompressor instance. */ @Override public Decompressor newDecompressor() { return new ZstdDecompressor(); diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java index c33ca1f4ff6e7..4ed6ba57545d0 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java +++ b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java @@ -29,6 +29,7 @@ public ZstdNoDictCodec(int compressionLevel) { super(Mode.ZSTDNODICT, compressionLevel); } + /** The name for this codec. */ @Override public String toString() { return getClass().getSimpleName(); diff --git a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index 61808191556f0..f5d5019092cc9 100644 --- a/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/sandbox/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -41,11 +41,13 @@ protected ZstdNoDictCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } + /** Creates a new compressor instance.*/ @Override public Compressor newCompressor() { return new ZstdCompressor(compressionLevel); } + /** Creates a new decompressor instance. */ @Override public Decompressor newDecompressor() { return new ZstdDecompressor();