update docker and opensearch to support flint acceleration indices

Signed-off-by: YANGDB <[email protected]>
YANG-DB · Jan 9, 2025 · 4919459 · 4919459
1 parent cc8acb9
commit 4919459
Show file tree

Hide file tree

Showing 10 changed files with 737 additions and 12 deletions.
diff --git a/docker/apache-spark-iceberg/.env b/docker/apache-spark-iceberg/.env
@@ -2,7 +2,10 @@ MASTER_UI_PORT=8080
 MASTER_PORT=7077
 UI_PORT=4040
 PPL_JAR=../../sparkPPLCosmetic/target/scala-2.12/opensearch-spark-ppl-assembly-0.7.0-SNAPSHOT.jar
-FLINT_JAR=../../sparkSqlApplicationCosmetic/target/scala-2.12/opensearch-spark-sql-application-assembly-0.7.0-SNAPSHOT.jar
+FLINT_JAR=../../flint-spark-integration/target/scala-2.12/flint-spark-integration-assembly-0.7.0-SNAPSHOT.jar
+OPENSEARCH_VERSION=latest
+DASHBOARDS_VERSION=latest
+
 OPENSEARCH_NODE_MEMORY=512m
 OPENSEARCH_ADMIN_PASSWORD=C0rrecthorsebatterystaple.
 OPENSEARCH_PORT=9200

diff --git a/docker/apache-spark-iceberg/docker-compose.yml b/docker/apache-spark-iceberg/docker-compose.yml
@@ -1,17 +1,17 @@
 version: "3"
 
 services:
-  spark-iceberg:
+  spark-tutorial:
     image: tabulario/spark-iceberg
-    container_name: spark-iceberg
+    container_name: spark-tutorial
     networks:
       iceberg_net:
     depends_on:
       - rest
       - minio
     volumes:
       - ./warehouse:/home/iceberg/warehouse
-      - ./notebooks:/home/iceberg/notebooks/notebooks
+      - ./notebooks:/home/iceberg/notebooks/PPL
       - type: bind
         source: ./spark-defaults.conf
         target: /opt/spark/conf/spark-defaults.conf
@@ -51,6 +51,8 @@ services:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
       - MINIO_DOMAIN=minio
+    volumes:
+      - minio-data:/data
     networks:
       iceberg_net:
         aliases:
@@ -78,5 +80,57 @@ services:
       /usr/bin/mc policy set public minio/warehouse;
       tail -f /dev/null
       "
+  opensearch:
+    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-latest}
+    container_name: opensearch
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch
+      - discovery.seed_hosts=opensearch
+      - cluster.initial_cluster_manager_nodes=opensearch
+      - bootstrap.memory_lock=true
+      - plugins.security.ssl.http.enabled=false
+      - OPENSEARCH_JAVA_OPTS=-Xms${OPENSEARCH_NODE_MEMORY:-512m} -Xmx${OPENSEARCH_NODE_MEMORY:-512m}
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_ADMIN_PASSWORD}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+      - ./opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+      - ./opensearch/security/config:/usr/share/opensearch/plugins/opensearch-security/securityconfig
+    ports:
+      - ${OPENSEARCH_PORT:-9200}:9200
+      - 9600:9600
+    expose:
+      - "${OPENSEARCH_PORT:-9200}"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "admin:${OPENSEARCH_ADMIN_PASSWORD}", "http://localhost:9200/_cluster/health"]
+      interval: 1m
+      timeout: 5s
+      retries: 3
+    networks:
+      iceberg_net:
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:${DASHBOARDS_VERSION}
+    container_name: opensearch-dashboards
+    ports:
+      - ${OPENSEARCH_DASHBOARDS_PORT:-5601}:5601
+    expose:
+      - "${OPENSEARCH_DASHBOARDS_PORT:-5601}"
+    environment:
+      OPENSEARCH_HOSTS: '["http://opensearch:9200"]'
+    depends_on:
+      - opensearch
+    networks:
+      iceberg_net:
 networks:
-  iceberg_net:
+  iceberg_net:
+
+volumes:
+  opensearch-data:
+  minio-data: