From 678be65f1aa649fae324d37fcadd38036fede158 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Wed, 17 Jan 2024 21:23:19 +0530 Subject: [PATCH 1/3] custom codecs upgrade to lucene99 codec Signed-off-by: Sarthak Aggarwal --- .../customcodecs/CustomCodecService.java | 8 +- .../customcodecs/Lucene95CustomCodec.java | 38 +----- .../customcodecs/Lucene99CustomCodec.java | 115 ++++++++++++++++++ ... => Lucene99CustomStoredFieldsFormat.java} | 71 ++++++----- .../index/codec/customcodecs/Zstd95Codec.java | 46 +++++++ ...dCodec.java => Zstd95DeprecatedCodec.java} | 28 +---- .../{ZstdCodec.java => Zstd99Codec.java} | 8 +- .../codec/customcodecs/ZstdNoDict95Codec.java | 45 +++++++ ...oDictCodec.java => ZstdNoDict99Codec.java} | 8 +- .../services/org.apache.lucene.codecs.Codec | 8 +- .../codec/customcodecs/CustomCodecTests.java | 48 ++++---- ...Lucene95CustomStoredFieldsFormatTests.java | 57 --------- ...Lucene99CustomStoredFieldsFormatTests.java | 57 +++++++++ 13 files changed, 355 insertions(+), 182 deletions(-) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java rename src/main/java/org/opensearch/index/codec/customcodecs/{Lucene95CustomStoredFieldsFormat.java => Lucene99CustomStoredFieldsFormat.java} (63%) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java rename src/main/java/org/opensearch/index/codec/customcodecs/{ZstdDeprecatedCodec.java => Zstd95DeprecatedCodec.java} (54%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ZstdCodec.java => Zstd99Codec.java} (86%) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java rename src/main/java/org/opensearch/index/codec/customcodecs/{ZstdNoDictCodec.java => ZstdNoDict99Codec.java} (85%) delete mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index de0eb2b..90519d2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -47,11 +47,11 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel)); } else { - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel)); } this.codecs = codecs.immutableMap(); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java index 89acc98..567d5e9 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java @@ -8,12 +8,9 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; +import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import java.util.Collections; import java.util.Set; @@ -22,6 +19,7 @@ * * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. * Supports two modes zstd and zstd_no_dict. + * Uses Lucene95 as the delegate codec * * @opensearch.internal */ @@ -73,40 +71,14 @@ public Set getAliases() { private final StoredFieldsFormat storedFieldsFormat; /** - * Creates a new compression codec with the default compression level. + * Creates a new compression codec. * * @param mode The compression codec (ZSTD or ZSTDNODICT). */ - public Lucene95CustomCodec(Mode mode) { - this(mode, DEFAULT_COMPRESSION_LEVEL); - } - /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @param compressionLevel The compression level. - */ - public Lucene95CustomCodec(Mode mode, int compressionLevel) { + public Lucene95CustomCodec(Mode mode) { super(mode.getCodec(), new Lucene95Codec()); - this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); - } - - /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. - */ - public Lucene95CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel); + this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(); } @Override diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java new file mode 100644 index 0000000..e0b3c2b --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; + +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene99 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene99CustomCodec extends FilterCodec { + + /** Default compression level used for compression */ + public static final int DEFAULT_COMPRESSION_LEVEL = 3; + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** + * ZStandard mode with dictionary + */ + ZSTD("ZSTD99", Set.of("zstd")), + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** + * Returns the Codec that is registered with Lucene + */ + public String getCodec() { + return codec; + } + + /** + * Returns the aliases of the Codec + */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + */ + public Lucene99CustomCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + */ + public Lucene99CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene99Codec()); + this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java similarity index 63% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java index 512df54..dac4853 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java @@ -22,10 +22,11 @@ import java.util.Objects; /** Stored field format used by pluggable codec */ -public class Lucene95CustomStoredFieldsFormat extends StoredFieldsFormat { +public class Lucene99CustomStoredFieldsFormat extends StoredFieldsFormat { /** A key that we use to map to a mode */ - public static final String MODE_KEY = Lucene95CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; + public static final String LUCENE95_MODE_KEY = "Lucene95CustomStoredFieldsFormat.mode"; + public static final String MODE_KEY = Lucene99CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; @@ -34,12 +35,12 @@ public class Lucene95CustomStoredFieldsFormat extends StoredFieldsFormat { private final CompressionMode zstdCompressionMode; private final CompressionMode zstdNoDictCompressionMode; - private final Lucene95CustomCodec.Mode mode; + private final Lucene99CustomCodec.Mode mode; private final int compressionLevel; /** default constructor */ - public Lucene95CustomStoredFieldsFormat() { - this(Lucene95CustomCodec.Mode.ZSTD, Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL); + public Lucene99CustomStoredFieldsFormat() { + this(Lucene99CustomCodec.Mode.ZSTD, Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL); } /** @@ -47,8 +48,8 @@ public Lucene95CustomStoredFieldsFormat() { * * @param mode The mode represents ZSTD or ZSTDNODICT */ - public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { - this(mode, Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL); + public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode) { + this(mode, Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL); } /** @@ -57,7 +58,7 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { * @param mode The mode represents ZSTD or ZSTDNODICT * @param compressionLevel The compression level for the mode. */ - public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compressionLevel) { + public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; zstdCompressionMode = new ZstdCompressionMode(compressionLevel); @@ -73,12 +74,17 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compr */ @Override public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { - String value = si.getAttribute(MODE_KEY); - if (value == null) { + if (si.getAttribute(LUCENE95_MODE_KEY) != null) { + String value = si.getAttribute(LUCENE95_MODE_KEY); + Lucene95CustomCodec.Mode mode = Lucene95CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else if (si.getAttribute(MODE_KEY) !=null){ + String value = si.getAttribute(MODE_KEY); + Lucene99CustomCodec.Mode mode = Lucene99CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); } - Lucene95CustomCodec.Mode mode = Lucene95CustomCodec.Mode.valueOf(value); - return impl(mode).fieldsReader(directory, si, fn, context); } /** @@ -98,31 +104,40 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo return impl(mode).fieldsWriter(directory, si, context); } + StoredFieldsFormat impl(Lucene99CustomCodec.Mode mode) { + switch (mode) { + case ZSTD: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); + case ZSTD_NO_DICT: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + default: + throw new AssertionError(); + } + } + StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { switch (mode) { case ZSTD: case ZSTD_DEPRECATED: - return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstd", - zstdCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT - ); + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); case ZSTD_NO_DICT: - return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstdNoDict", - zstdNoDictCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT - ); + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); default: throw new AssertionError(); } } - public Lucene95CustomCodec.Mode getMode() { + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { + return new Lucene90CompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT + ); + } + + public Lucene99CustomCodec.Mode getMode() { return mode; } @@ -134,7 +149,7 @@ public int getCompressionLevel() { } public CompressionMode getCompressionMode() { - return mode == Lucene95CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + return mode == Lucene99CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java new file mode 100644 index 0000000..8cbf1e5 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** + * ZstdCodec provides ZSTD compressor using the zstd-jni library. + */ +public class Zstd95Codec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { + + /** + * Creates a new ZstdCodec instance with the default compression level. + */ + public Zstd95Codec() { + super(Mode.ZSTD); + } + + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95DeprecatedCodec.java similarity index 54% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Zstd95DeprecatedCodec.java index 02fa386..c59a2da 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95DeprecatedCodec.java @@ -8,44 +8,22 @@ package org.opensearch.index.codec.customcodecs; -import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; /** * ZstdDeprecatedCodec provides ZSTD compressor using the zstd-jni library. * Added to support backward compatibility for indices created with Lucene95CustomCodec as codec name. */ @Deprecated(since = "2.10") -public class ZstdDeprecatedCodec extends Lucene95CustomCodec implements CodecSettings { +public class Zstd95DeprecatedCodec extends Lucene95CustomCodec implements CodecSettings { /** * Creates a new ZstdDefaultCodec instance with the default compression level. */ - public ZstdDeprecatedCodec() { - this(DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new ZstdDefaultCodec instance. - * - * @param compressionLevel The compression level. - */ - public ZstdDeprecatedCodec(int compressionLevel) { - super(Mode.ZSTD_DEPRECATED, compressionLevel); - } - - /** - * Creates a new ZstdDefaultCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public ZstdDeprecatedCodec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD_DEPRECATED, compressionLevel, mapperService, logger); + public Zstd95DeprecatedCodec() { + super(Mode.ZSTD_DEPRECATED); } /** The name for this codec. */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java similarity index 86% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java index a3e3a34..df86806 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java @@ -20,12 +20,12 @@ /** * ZstdCodec provides ZSTD compressor using the zstd-jni library. */ -public class ZstdCodec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { +public class Zstd99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { /** * Creates a new ZstdCodec instance with the default compression level. */ - public ZstdCodec() { + public Zstd99Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -34,7 +34,7 @@ public ZstdCodec() { * * @param compressionLevel The compression level. */ - public ZstdCodec(int compressionLevel) { + public Zstd99Codec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } @@ -45,7 +45,7 @@ public ZstdCodec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public ZstdCodec(MapperService mapperService, Logger logger, int compressionLevel) { + public Zstd99Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD, compressionLevel, mapperService, logger); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java new file mode 100644 index 0000000..abf9e56 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** + * ZstdNoDictCodec provides ZSTD compressor without a dictionary support. + */ +public class ZstdNoDict95Codec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { + + /** + * Creates a new ZstdNoDictCodec instance with the default compression level. + */ + public ZstdNoDict95Codec() { + super(Mode.ZSTD_NO_DICT); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD_NO_DICT.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java similarity index 85% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java index ea7351f..1d65aee 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java @@ -20,12 +20,12 @@ /** * ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ -public class ZstdNoDictCodec extends Lucene95CustomCodec implements CodecSettings, CodecAliases { +public class ZstdNoDict99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { /** * Creates a new ZstdNoDictCodec instance with the default compression level. */ - public ZstdNoDictCodec() { + public ZstdNoDict99Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -34,7 +34,7 @@ public ZstdNoDictCodec() { * * @param compressionLevel The compression level. */ - public ZstdNoDictCodec(int compressionLevel) { + public ZstdNoDict99Codec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } @@ -45,7 +45,7 @@ public ZstdNoDictCodec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public ZstdNoDictCodec(MapperService mapperService, Logger logger, int compressionLevel) { + public ZstdNoDict99Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); } diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index ba50540..b823485 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,3 +1,5 @@ -org.opensearch.index.codec.customcodecs.ZstdCodec -org.opensearch.index.codec.customcodecs.ZstdNoDictCodec -org.opensearch.index.codec.customcodecs.ZstdDeprecatedCodec +org.opensearch.index.codec.customcodecs.Zstd95Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict95Codec +org.opensearch.index.codec.customcodecs.Zstd95DeprecatedCodec +org.opensearch.index.codec.customcodecs.Zstd99Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict99Codec \ No newline at end of file diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index 5365b9e..9cc3157 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -81,16 +81,16 @@ public void setup() { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); + Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdDeprecatedCodec() { @@ -104,16 +104,16 @@ public void testZstdDeprecatedCodec() { public void testZstdWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); + Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } @@ -154,41 +154,41 @@ public void testZstandardCompressionLevelSupport() throws Exception { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); } public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); + Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); } - private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene95CustomStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene99CustomStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene95CustomCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene99CustomCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java deleted file mode 100644 index e669b7f..0000000 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.test.OpenSearchTestCase; - -public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { - - public void testDefaultLucene95CustomCodecMode() { - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(); - assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); - } - - public void testZstdNoDictLucene95CustomCodecMode() { - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT - ); - assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); - } - - public void testZstdModeWithCompressionLevel() { - int randomCompressionLevel = randomIntBetween(1, 6); - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD, - randomCompressionLevel - ); - assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); - assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { - int randomCompressionLevel = randomIntBetween(1, 6); - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT, - randomCompressionLevel - ); - assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); - assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); - } - - public void testCompressionModes(){ - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(); - assertTrue(lucene95CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); - } - - public void testZstdNoDictCompressionModes(){ - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode.ZSTD_NO_DICT); - assertTrue(lucene95CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); - } - -} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000..c966901 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene99CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene99CustomCodecMode() { + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(); + assertEquals(Lucene99CustomCodec.Mode.ZSTD, lucene99CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene99CustomCodecMode() { + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat( + Lucene99CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, lucene99CustomStoredFieldsFormat.getMode()); + } + + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat( + Lucene99CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene99CustomCodec.Mode.ZSTD, lucene99CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene99CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene99CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat( + Lucene99CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, lucene99CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene99CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testCompressionModes(){ + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(); + assertTrue(lucene99CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); + } + + public void testZstdNoDictCompressionModes(){ + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode.ZSTD_NO_DICT); + assertTrue(lucene99CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); + } + +} From eae6261aa61b55c67a0ae12b5d56401567f8f0de Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Tue, 23 Jan 2024 18:23:00 +0530 Subject: [PATCH 2/3] moving lucene95 extended codecs to backward_codecs Signed-off-by: Sarthak Aggarwal --- .../customcodecs/Lucene99CustomStoredFieldsFormat.java | 1 + .../{ => backward_codecs}/Lucene95CustomCodec.java | 3 ++- .../customcodecs/{ => backward_codecs}/Zstd95Codec.java | 2 +- .../{ => backward_codecs}/Zstd95DeprecatedCodec.java | 2 +- .../{ => backward_codecs}/ZstdNoDict95Codec.java | 2 +- .../META-INF/services/org.apache.lucene.codecs.Codec | 6 +++--- 6 files changed, 9 insertions(+), 7 deletions(-) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs}/Lucene95CustomCodec.java (94%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs}/Zstd95Codec.java (94%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs}/Zstd95DeprecatedCodec.java (94%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs}/ZstdNoDict95Codec.java (94%) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java index dac4853..59dde27 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java @@ -17,6 +17,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomCodec; import java.io.IOException; import java.util.Objects; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java similarity index 94% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java index 567d5e9..e05f08d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java @@ -6,11 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; +import org.opensearch.index.codec.customcodecs.Lucene99CustomStoredFieldsFormat; import java.util.Collections; import java.util.Set; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java similarity index 94% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java index 8cbf1e5..e41f690 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95DeprecatedCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95DeprecatedCodec.java similarity index 94% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd95DeprecatedCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95DeprecatedCodec.java index c59a2da..61e16ad 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd95DeprecatedCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95DeprecatedCodec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecSettings; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/ZstdNoDict95Codec.java similarity index 94% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/ZstdNoDict95Codec.java index abf9e56..f85753d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict95Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/ZstdNoDict95Codec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index b823485..5912ede 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,5 +1,5 @@ -org.opensearch.index.codec.customcodecs.Zstd95Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict95Codec -org.opensearch.index.codec.customcodecs.Zstd95DeprecatedCodec +org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95Codec +org.opensearch.index.codec.customcodecs.backward_codecs.ZstdNoDict95Codec +org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95DeprecatedCodec org.opensearch.index.codec.customcodecs.Zstd99Codec org.opensearch.index.codec.customcodecs.ZstdNoDict99Codec \ No newline at end of file From 85f73d47157c7e3ab107f1ea2a647b5fd6ed4329 Mon Sep 17 00:00:00 2001 From: mgodwan Date: Thu, 25 Jan 2024 17:51:38 +0530 Subject: [PATCH 3/3] Trial: bwc test enablement Signed-off-by: mgodwan --- .github/workflows/test_bwc.yml | 60 ++++++ build.gradle | 180 ++++++++++++++++++ scripts/bwctest.sh | 31 +++ .../codec/customcodecs/bwc/SampleBWCIT.java | 83 ++++++++ 4 files changed, 354 insertions(+) create mode 100644 .github/workflows/test_bwc.yml create mode 100644 scripts/bwctest.sh create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/bwc/SampleBWCIT.java diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml new file mode 100644 index 0000000..2bd831f --- /dev/null +++ b/.github/workflows/test_bwc.yml @@ -0,0 +1,60 @@ +name: Test Custom Codecs BWC +on: + push: + branches: + - "*" + pull_request: + branches: + - "*" + +jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + Build-custom-codecs-linux: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: [11,17,21] + fail-fast: false + + name: Test Custom Codecs BWC + runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + + steps: + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + # custom-codecs + - name: Checkout Custom codecs + uses: actions/checkout@v3 + + - name: Assemble custom-codecs + run: | + plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'` + chown -R 1000:1000 `pwd` + echo plugin_version $plugin_version + su `id -un 1000` -c "./gradlew assemble" + echo "Creating ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/custom-codecs/$plugin_version ..." + su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version" + echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version ..." + ls ./build/distributions/ + su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version" + echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version ..." + ls ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version + - name: Run Custom Codecs Backwards Compatibility Tests + run: | + echo "Running backwards compatibility tests ..." + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false --debug" diff --git a/build.gradle b/build.gradle index 00de61c..afacdc7 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,6 @@ +import java.util.concurrent.Callable +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask + /* * SPDX-License-Identifier: Apache-2.0 * @@ -15,6 +18,17 @@ buildscript { opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") + + bwcVersionShort = "2.11.0" + bwcVersion = bwcVersionShort + ".0" + //https://github.com/opensearch-project/custom-codecs/archive/refs/tags/2.11.0.0.zip +// bwcOpenSearchCustomCodecsDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' + +// 'opensearch/plugins/custom-codecs-' + bwcVersion + '.zip' + //https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-custom-codecs/2.11.0.0/opensearch-custom-codecs-2.11.0.0.zip + bwcOpenSearchCustomCodecsDownload = 'https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-custom-codecs/' + bwcVersion + '/opensearch-custom-codecs-' + bwcVersion + '.zip' + baseName = "customCodecsBwcCluster" + bwcFilePath = "src/test/resources/org/opensearch/customcodecs/bwc/" + bwcCustomCodecsPath = bwcFilePath + "custom-codecs/" } repositories { @@ -185,6 +199,12 @@ integTest { systemProperty "https", System.getProperty("https") systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") + + if (System.getProperty("tests.rest.bwcsuite") == null) { + filter { + excludeTestsMatching "org.opensearch.index.codecs.customcodecs.bwc.*IT" + } + } } testClusters.integTest { @@ -192,6 +212,166 @@ testClusters.integTest { plugin(project.tasks.bundlePlugin.archiveFile) } +2.times {i -> + testClusters { + "${baseName}$i" { + testDistribution = "ARCHIVE" + versions = [bwcVersionShort, opensearch_version] + numberOfNodes = 3 + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + if (new File("$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion").exists()) { + project.delete(files("$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion")) + } + project.mkdir bwcCustomCodecsPath + bwcVersion + ant.get(src: bwcOpenSearchCustomCodecsDownload, + dest: "$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion", + httpusecaches: false) + return fileTree("$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion").getSingleFile() + } + } + } + })) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' + } + } +} + +String opensearchCustomCodecsPlugin = "opensearch-custom-codecs-" + project.version + ".zip" +List> plugins = [ + provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + project.mkdir "$bwcFilePath/$project.version" + copy { + from "$buildDir/distributions/$opensearchCustomCodecsPlugin" + into "$bwcFilePath/$project.version" + } + return fileTree(bwcFilePath + project.version).getSingleFile() + } + } + } + }) +] + +// Creates 2 test clusters with 3 nodes of the old version. +2.times {i -> + task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}$i" + filter { + includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*" + } + systemProperty 'tests.rest.bwcsuite', 'old_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'old' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") + } +} + +// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +// This is also used as a one third upgraded cluster for a rolling upgrade. +task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}0" + dependsOn "${baseName}#oldVersionClusterTask0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*" + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'first' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. +// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +// This is used for rolling upgrade. +task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#mixedClusterTask" + useCluster testClusters."${baseName}0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*" + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'second' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. +// This results in a fully upgraded cluster. +// This is used for rolling upgrade. +task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#twoThirdsUpgradedClusterTask" + useCluster testClusters."${baseName}0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*" + } + mustRunAfter "${baseName}#mixedClusterTask" + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'third' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version +// at the same time resulting in a fully upgraded cluster. +task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oldVersionClusterTask1" + useCluster testClusters."${baseName}1" + doFirst { + testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*" + } + systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") +} + +// A bwc test suite which runs all the bwc tasks combined. +task bwcTestSuite(type: StandaloneRestIntegTestTask) { + exclude '**/*Test*' + exclude '**/*IT*' + dependsOn tasks.named("${baseName}#mixedClusterTask") + dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") + dependsOn tasks.named("${baseName}#fullRestartClusterTask") +} + +run { + doFirst { + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + useCluster testClusters.integTest +} + tasks.withType(PublishToMavenRepository) { def predicate = provider { publication.name == "pluginZip" diff --git a/scripts/bwctest.sh b/scripts/bwctest.sh new file mode 100644 index 0000000..6361e38 --- /dev/null +++ b/scripts/bwctest.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +function usage() { + echo "" + echo "This script is used to run Backwards Compatibility tests" + echo "--------------------------------------------------------------------------" + echo "Usage: $0 [args]" + echo "" + echo "Required arguments:" + echo "None" + echo "" + echo -e "-h\tPrint this message." + echo "--------------------------------------------------------------------------" +} + +while getopts ":h" arg; do + case $arg in + h) + usage + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}" + exit 1 + ;; + esac +done + +./gradlew bwcTestSuite -Dtests.security.manager=false \ No newline at end of file diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/bwc/SampleBWCIT.java b/src/test/java/org/opensearch/index/codec/customcodecs/bwc/SampleBWCIT.java new file mode 100644 index 0000000..6b8c1c6 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/bwc/SampleBWCIT.java @@ -0,0 +1,83 @@ +package org.opensearch.index.codec.customcodecs.bwc; + +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class SampleBWCIT extends OpenSearchRestTestCase { + + private static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite")); +// private static final String CLUSTER_NAME = System.getProperty("tests.clustername"); +// private static final String MIXED_CLUSTER_TEST_ROUND = System.getProperty("tests.rest.bwcsuite_round"); + private String dataIndexName = "test_data_for_ad_plugin"; + private int detectionIntervalInMinutes = 1; + private int windowDelayIntervalInMinutes = 1; + private String aggregationMethod = "sum"; + private int totalDocsPerCategory = 10_000; + private int categoryFieldSize = 2; + private List runningRealtimeDetectors; + private List historicalDetectors; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + this.runningRealtimeDetectors = new ArrayList<>(); + this.historicalDetectors = new ArrayList<>(); + } + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings + .builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") + .build(); + } + + private enum ClusterType { + OLD, + MIXED, + UPGRADED; + + public static ClusterType parse(String value) { + switch (value) { + case "old_cluster": + return OLD; + case "mixed_cluster": + return MIXED; + case "upgraded_cluster": + return UPGRADED; + default: + throw new AssertionError("unknown cluster type: " + value); + } + } + } + + + @SuppressWarnings("unchecked") + public void testBackwardsCompatibility() throws Exception { + logger.info("Cluster type ::: -> " + CLUSTER_TYPE); + } +}