Upgrade search versions (open-metadata#13479)

* working on maven shading * update es in docker * add maven shade for elasticsearch/opensearch dependencies * add submodule for es/os dependencies * upgrade docker elasticsearch to latest 8.10.2 * upgrade docker elasticsearch to latest 8.10.2 * Update Postgres dev to use OpenSearch * Fix container service dependency * revert conf change --------- Co-authored-by: 07Himank <[email protected]> Co-authored-by: 07Himank <[email protected]>
gorgias · Oct 8, 2023 · 35c4e18 · 35c4e18
1 parent b06db04
commit 35c4e18
Show file tree

Hide file tree

Showing 57 changed files with 839 additions and 479 deletions.
diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml
@@ -24,7 +24,7 @@ services:
     restart: always
     command: "--work_mem=10MB"
     depends_on:
-      - elasticsearch
+      - opensearch
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: password
@@ -42,12 +42,13 @@ services:
       timeout: 10s
       retries: 10
 
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
-    container_name: openmetadata_elasticsearch
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    container_name: openmetadata_opensearch
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
+      - plugins.security.disabled=true
     networks:
       - local_app_net
     expose:
@@ -57,7 +58,7 @@ services:
       - "9200:9200"
       - "9300:9300"
     volumes:
-      - es-data:/usr/share/elasticsearch/data
+      - es-data:/usr/share/opensearch/data
 
   execute-migrate-all:
     build:
@@ -113,7 +114,7 @@ services:
       DB_PORT: ${DB_PORT:-5432}
       OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
       # ElasticSearch Configurations
-      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
+      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- opensearch}
       ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
       ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
       ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
@@ -211,7 +212,7 @@ services:
       WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
       WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
     depends_on:
-      elasticsearch:
+      opensearch:
         condition: service_started
       postgresql:
         condition: service_healthy
@@ -344,7 +345,7 @@ services:
       - "8586:8586"
       - "5005:5005"
     depends_on:
-      elasticsearch:
+      ingestion:
         condition: service_started
       postgresql:
         condition: service_healthy
@@ -376,7 +377,7 @@ services:
     command:
       - "/opt/airflow/ingestion_dependency.sh"
     depends_on:
-      elasticsearch:
+      opensearch:
         condition: service_started
       postgresql:
         condition: service_healthy

diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml
@@ -42,11 +42,12 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
+      - xpack.security.enabled=false
     networks:
       - local_app_net
     expose:

diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml
@@ -41,10 +41,11 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
+      - xpack.security.enabled=false
     networks:
       - app_net
     ports:

diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml
@@ -39,10 +39,11 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
+      - xpack.security.enabled=false
     networks:
       - app_net
     ports:

diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml
@@ -39,6 +39,16 @@
       <artifactId>common</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.open-metadata</groupId>
+      <artifactId>elasticsearch-deps</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.open-metadata</groupId>
+      <artifactId>opensearch-deps</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.antlr</groupId>
       <artifactId>antlr4-runtime</artifactId>
@@ -129,57 +139,6 @@
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-analyzers-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-backward-codecs</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-grouping</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-memory</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-spatial</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-spatial-extras</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-spatial3d</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.lucene</groupId>
-          <artifactId>lucene-suggest</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.fasterxml.jackson.dataformat</groupId>
-          <artifactId>jackson-dataformat-cbor</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.yaml</groupId>
-          <artifactId>snakeyaml</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.opensearch.client</groupId>
-      <artifactId>opensearch-rest-high-level-client</artifactId>
-      <version>${opensearch.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpasyncclient</artifactId>
@@ -769,6 +728,153 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-source-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>3.5.1</version> <!-- Use the latest version available -->
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <shadedArtifactId>my-app-shaded</shadedArtifactId>
+              <shadedArtifactAttached>true</shadedArtifactAttached>
+              <shadedClassifierName>shaded</shadedClassifierName>
+              <relocations>
+                <!-- Relocate Elasticsearch's Lucene -->
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-core</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-analyzers-common</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-backward-codecs</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-grouping</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-highlighter</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-join</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-memory</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-misc</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-queries</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-queryparser</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-sandbox</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-spatial3d</shadedPattern>
+                  </relocation>
+                  <relocation>
+                    <pattern>org.apache.lucene</pattern>
+                    <shadedPattern>es.lucene-suggest</shadedPattern>
+                  </relocation>
+                <!-- Relocate OpenSearch's Lucene -->
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-core</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-analysis-common</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-backward-codecs</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-grouping</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-highlighter</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-join</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-memory</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-misc</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-queries</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-queryparser</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-sandbox</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-spatial-extras</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-spatial3d</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache.lucene</pattern>
+                  <shadedPattern>os.lucene-suggest</shadedPattern>
+                </relocation>
+              </relocations>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>module-info.class</exclude>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <transformers>
+                <!-- add Main-Class to manifest file -->
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                  <mainClass>org.openmetadata.service.OpenMetadataApplication</mainClass>
+                </transformer>
+              </transformers>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
@@ -16,6 +16,8 @@
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.service.search.SearchRepository.ELASTIC_SEARCH_EXTENSION;
 
+import es.org.elasticsearch.action.search.SearchResponse;
+import es.org.elasticsearch.search.suggest.Suggest;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Content;
@@ -40,8 +42,6 @@
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.suggest.Suggest;
 import org.openmetadata.schema.api.CreateEventPublisherJob;
 import org.openmetadata.schema.system.EventPublisherJob;
 import org.openmetadata.service.Entity;

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
@@ -17,9 +17,9 @@
 import org.openmetadata.service.exception.CustomExceptionMessage;
 import org.openmetadata.service.search.models.IndexMapping;
 import org.openmetadata.service.util.SSLUtil;
-import org.opensearch.action.bulk.BulkRequest;
-import org.opensearch.action.bulk.BulkResponse;
-import org.opensearch.client.RequestOptions;
+import os.org.opensearch.action.bulk.BulkRequest;
+import os.org.opensearch.action.bulk.BulkResponse;
+import os.org.opensearch.client.RequestOptions;
 
 public interface SearchClient {
 
@@ -110,8 +110,8 @@ default BulkResponse bulk(BulkRequest data, RequestOptions options) throws IOExc
     throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
   }
 
-  default org.elasticsearch.action.bulk.BulkResponse bulk(
-      org.elasticsearch.action.bulk.BulkRequest data, org.elasticsearch.client.RequestOptions options)
+  default es.org.elasticsearch.action.bulk.BulkResponse bulk(
+      es.org.elasticsearch.action.bulk.BulkRequest data, es.org.elasticsearch.client.RequestOptions options)
       throws IOException {
     throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
   }
@@ -120,7 +120,7 @@ default int getSuccessFromBulkResponse(BulkResponse response) {
     throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
   }
 
-  default int getSuccessFromBulkResponse(org.elasticsearch.action.bulk.BulkResponse response) {
+  default int getSuccessFromBulkResponse(es.org.elasticsearch.action.bulk.BulkResponse response) {
     throw new CustomExceptionMessage(Response.Status.NOT_IMPLEMENTED, NOT_IMPLEMENTED_METHOD);
   }