diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index aede58e41..2f2134947 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -18,7 +18,7 @@ jobs: needs: Get-CI-Image-Tag strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] # Job name name: Build and test Security Analytics on linux # This job runs on Linux diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 725dd9207..9750d6976 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -12,7 +12,7 @@ jobs: build: strategy: matrix: - java: [ 11, 17 ] + java: [ 11, 17, 21 ] # Job name name: Build and test SecurityAnalytics # This job runs on Linux diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 050308f98..7b2552ed0 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -57,9 +57,8 @@ When launching a cluster using one of the above commands, logs are placed in `bu #### Building from the IDE -Currently, the only IDE we support is IntelliJ IDEA. It's free, it's open source, it works. The gradle tasks above can also be launched from IntelliJ's Gradle toolbar and the extra parameters can be passed in via the Launch Configurations VM arguments. +Currently, the only IDE we support is IntelliJ IDEA. It's free, it's open source, it works. The gradle tasks above can also be launched from IntelliJ's Gradle toolbar and the extra parameters can be passed in via the Launch Configurations VM arguments. ### Backport - [Link to backport documentation](https://github.com/opensearch-project/opensearch-plugins/blob/main/BACKPORT.md) - diff --git a/build.gradle b/build.gradle index 681287dbb..7a1a85aef 100644 --- a/build.gradle +++ b/build.gradle @@ -37,8 +37,8 @@ buildscript { plugins { id 'java' - id 'com.diffplug.spotless' version '6.20.0' - id "com.netflix.nebula.ospackage" version "11.3.0" + id 'com.diffplug.spotless' version '6.22.0' + id "com.netflix.nebula.ospackage" version "11.5.0" id 'java-library' } @@ -143,6 +143,13 @@ repositories { sourceSets.main.java.srcDirs = ['src/main/generated','src/main/java'] configurations { zipArchive + + all { + resolutionStrategy { + // for spotless transitive dependency CVE + force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0" + } + } } dependencies { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a79e..d64cd4917 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index daf8a3998..550b58023 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=03ec176d388f2aa99defcadc3ac6adf8dd2bce5145a129659537c0874dea5ad1 +distributionSha256Sum=9d926787066a081739e8200858338b4a69e837c3a821a33aca9db09dd4a41026 diff --git a/gradlew b/gradlew index 325571e39..1aa94a426 100755 --- a/gradlew +++ b/gradlew @@ -83,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,6 +134,7 @@ else if ! command -v java >/dev/null 2>&1 then die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi @@ -143,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -151,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -200,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java index 2e6091572..07721ae9b 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/CorrelationCodecVersion.java @@ -5,10 +5,14 @@ package org.opensearch.securityanalytics.correlation.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.index.mapper.MapperService; -import org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec; -import org.opensearch.securityanalytics.correlation.index.codec.correlation950.PerFieldCorrelationVectorsFormat; +import org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950; +import org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990; +import org.opensearch.securityanalytics.correlation.index.codec.correlation990.PerFieldCorrelationVectorsFormat990; +import org.opensearch.securityanalytics.correlation.index.codec.correlation950.PerFieldCorrelationVectorsFormat950; import java.util.Optional; import java.util.function.BiFunction; @@ -16,28 +20,35 @@ public enum CorrelationCodecVersion { V_9_5_0( - "CorrelationCodec", + "CorrelationCodec950", new Lucene95Codec(), - new PerFieldCorrelationVectorsFormat(Optional.empty()), - (userCodec, mapperService) -> new CorrelationCodec(userCodec, new PerFieldCorrelationVectorsFormat(Optional.of(mapperService))), - CorrelationCodec::new + new PerFieldCorrelationVectorsFormat950(Optional.empty()), + (userCodec, mapperService) -> new CorrelationCodec950(userCodec, new PerFieldCorrelationVectorsFormat950(Optional.of(mapperService))), + CorrelationCodec950::new + ), + V_9_9_0( + "CorrelationCodec990", + new Lucene99Codec(), + new PerFieldCorrelationVectorsFormat990(Optional.empty()), + (userCodec, mapperService) -> new CorrelationCodec990(userCodec, new PerFieldCorrelationVectorsFormat990(Optional.of(mapperService))), + CorrelationCodec990::new ); - private static final CorrelationCodecVersion CURRENT = V_9_5_0; + private static final CorrelationCodecVersion CURRENT = V_9_9_0; private final String codecName; private final Codec defaultCodecDelegate; - private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat; + private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat; private final BiFunction correlationCodecSupplier; private final Supplier defaultCorrelationCodecSupplier; CorrelationCodecVersion(String codecName, Codec defaultCodecDelegate, - PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat, + PerFieldKnnVectorsFormat perFieldKnnVectorsFormat, BiFunction correlationCodecSupplier, Supplier defaultCorrelationCodecSupplier) { this.codecName = codecName; this.defaultCodecDelegate = defaultCodecDelegate; - this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat; + this.perFieldKnnVectorsFormat = perFieldKnnVectorsFormat; this.correlationCodecSupplier = correlationCodecSupplier; this.defaultCorrelationCodecSupplier = defaultCorrelationCodecSupplier; } @@ -50,8 +61,8 @@ public Codec getDefaultCodecDelegate() { return defaultCodecDelegate; } - public PerFieldCorrelationVectorsFormat getPerFieldCorrelationVectorsFormat() { - return perFieldCorrelationVectorsFormat; + public PerFieldKnnVectorsFormat getPerFieldCorrelationVectorsFormat() { + return perFieldKnnVectorsFormat; } public BiFunction getCorrelationCodecSupplier() { diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/CorrelationCodec.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/CorrelationCodec950.java similarity index 71% rename from src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/CorrelationCodec.java rename to src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/CorrelationCodec950.java index 87f0ce8f1..2b7a71e60 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/CorrelationCodec.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/CorrelationCodec950.java @@ -7,17 +7,18 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecVersion; -public class CorrelationCodec extends FilterCodec { +public class CorrelationCodec950 extends FilterCodec { private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_5_0; - private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat; + private final PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat; - public CorrelationCodec() { + public CorrelationCodec950() { this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldCorrelationVectorsFormat()); } - public CorrelationCodec(Codec delegate, PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat) { + public CorrelationCodec950(Codec delegate, PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat) { super(VERSION.getCodecName(), delegate); this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat; } diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/PerFieldCorrelationVectorsFormat950.java similarity index 71% rename from src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java rename to src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/PerFieldCorrelationVectorsFormat950.java index 239e66ade..6fa602f30 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation950/PerFieldCorrelationVectorsFormat950.java @@ -4,15 +4,15 @@ */ package org.opensearch.securityanalytics.correlation.index.codec.correlation950; -import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat; +import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.securityanalytics.correlation.index.codec.BasePerFieldCorrelationVectorsFormat; import java.util.Optional; -public class PerFieldCorrelationVectorsFormat extends BasePerFieldCorrelationVectorsFormat { +public class PerFieldCorrelationVectorsFormat950 extends BasePerFieldCorrelationVectorsFormat { - public PerFieldCorrelationVectorsFormat(final Optional mapperService) { + public PerFieldCorrelationVectorsFormat950(final Optional mapperService) { super( mapperService, Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN, diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation990/CorrelationCodec990.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation990/CorrelationCodec990.java new file mode 100644 index 000000000..13821e3ef --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation990/CorrelationCodec990.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.index.codec.correlation990; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecVersion; + +public class CorrelationCodec990 extends FilterCodec { + private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_9_0; + private final PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat; + + public CorrelationCodec990() { + this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldCorrelationVectorsFormat()); + } + + public CorrelationCodec990(Codec delegate, PerFieldKnnVectorsFormat perFieldCorrelationVectorsFormat) { + super(VERSION.getCodecName(), delegate); + this.perFieldCorrelationVectorsFormat = perFieldCorrelationVectorsFormat; + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return perFieldCorrelationVectorsFormat; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation990/PerFieldCorrelationVectorsFormat990.java b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation990/PerFieldCorrelationVectorsFormat990.java new file mode 100644 index 000000000..15e78033e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/correlation/index/codec/correlation990/PerFieldCorrelationVectorsFormat990.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.securityanalytics.correlation.index.codec.correlation990; + +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.securityanalytics.correlation.index.codec.BasePerFieldCorrelationVectorsFormat; + +import java.util.Optional; + +public class PerFieldCorrelationVectorsFormat990 extends BasePerFieldCorrelationVectorsFormat { + + public PerFieldCorrelationVectorsFormat990(final Optional mapperService) { + super( + mapperService, + Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, + Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + () -> new Lucene99HnswVectorsFormat(), + (maxConn, beamWidth) -> new Lucene99HnswVectorsFormat(maxConn, beamWidth) + ); + } +} \ No newline at end of file diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 89f846a4d..e4a14ac56 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1 +1,2 @@ -org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec \ No newline at end of file +org.opensearch.securityanalytics.correlation.index.codec.correlation950.CorrelationCodec950 +org.opensearch.securityanalytics.correlation.index.codec.correlation990.CorrelationCodec990 \ No newline at end of file