Hudi quickstart

.github/workflows/test_with_hudi.yml

@@ -0,0 +1,81 @@
+name: Test with Hudi
+
+on:
+  schedule:
+    - cron: "5 9 * * 1"
+  push:
+    branches: [ main ]
+    paths:
+      - 'ci/**/quickstart/hudi/*'
+      - '.github/workflows/test_with_hudi.yml'
+      - 'quickstart_hudi_test.go'
+      - 'helper.go'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'ci/**/quickstart/hudi/*'
+      - '.github/workflows/test_with_hudi.yml'
+      - 'quickstart_hudi_test.go'
+      - 'helper.go'
+
+jobs:
+  build:
+
+    name: Build and test
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout the repo as this CI needs:
+      # - the compose file for StarRocks and Ginkgo/Gomega
+      - name: Checkout Test repo
+        uses: actions/checkout@v4
+        with:
+          path: testing
+
+      - name: Checkout Demo repo
+        uses: actions/checkout@v4
+        with:
+          repository: StarRocks/demo
+          path: demo
+
+      - name: Set up Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'testing/ci/go.mod'
+
+      - name: Install ginkgo
+        run: |
+          version=$(cat go.mod| grep "ginkgo/v2" | awk '{print $2}')
+          go install -v github.com/onsi/ginkgo/v2/ginkgo@$version
+        working-directory: ./testing/ci
+
+      - name: Start Hudi, StarRocks, and MinIO
+        run: docker compose up --detach --wait --wait-timeout 60
+        working-directory: demo/documentation-samples/hudi
+
+      - name: Create bucket
+        run: docker compose exec mc mc mb minio/huditest
+        working-directory: demo/documentation-samples/hudi
+
+      - name: Copy Spark script
+        run: |
+          docker compose cp ../../../testing/ci/SHELL/quickstart/hudi/spark_shell.scala spark-hudi:/spark/
+        working-directory: demo/documentation-samples/hudi
+
+      - name: Run Spark job
+        run: docker compose exec spark-hudi spark-shell -i /spark/spark_shell.scala
+        working-directory: demo/documentation-samples/hudi
+
+      # Any tests that will run against the StarRocks env would be 
+      # launched in steps like this one. Make sure to reset the 
+      # StarRocks environment after each run (remove any tables
+      # and databases created, and reset any settings to the default)
+      #
+      # The ginkgo command uses `--focus-file` to run only the one test
+      # file.
+      - name: Test; Hudi SQL test
+        if: always()
+        env:
+          SR_FE_HOST: 'localhost'
+        run: ginkgo -v --focus-file=./quickstart_hudi_test.go
+        working-directory: testing/ci

.github/workflows/test_with_oneFE_oneBE.yml

@@ -1,4 +1,4 @@
-name: Test the docs
+name: Test with one FE and one BE
 
 on: 
   workflow_dispatch:

ci/SHELL/quickstart/hudi/spark_shell.scala

@@ -0,0 +1,43 @@
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.SaveMode._
+import org.apache.hudi.DataSourceReadOptions._
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.config.HoodieWriteConfig._
+import scala.collection.JavaConversions._
+
+val schema = StructType( Array(
+                 StructField("language", StringType, true),
+                 StructField("users", StringType, true),
+                 StructField("id", StringType, true)
+             ))
+
+val rowData= Seq(Row("Java", "20000", "a"),
+               Row("Python", "100000", "b"),
+               Row("Scala", "3000", "c"))
+
+
+val df = spark.createDataFrame(rowData,schema)
+
+val databaseName = "hudi_sample"
+val tableName = "hudi_coders_hive"
+val basePath = "s3a://huditest/hudi_coders"
+
+df.write.format("hudi").
+  option(org.apache.hudi.config.HoodieWriteConfig.TABLE_NAME, tableName).
+  option(RECORDKEY_FIELD_OPT_KEY, "id").
+  option(PARTITIONPATH_FIELD_OPT_KEY, "language").
+  option(PRECOMBINE_FIELD_OPT_KEY, "users").
+  option("hoodie.datasource.write.hive_style_partitioning", "true").
+  option("hoodie.datasource.hive_sync.enable", "true").
+  option("hoodie.datasource.hive_sync.mode", "hms").
+  option("hoodie.datasource.hive_sync.database", databaseName).
+  option("hoodie.datasource.hive_sync.table", tableName).
+  option("hoodie.datasource.hive_sync.partition_fields", "language").
+  option("hoodie.datasource.hive_sync.partition_extractor_class", "org.apache.hudi.hive.MultiPartKeysValueExtractor").
+  option("hoodie.datasource.hive_sync.metastore.uris", "thrift://hive-metastore:9083").
+  mode(Overwrite).
+  save(basePath)
+System.exit(0)
+

ci/SQL/quickstart/hudi/create_catalog.sql

@@ -0,0 +1,14 @@
+CREATE EXTERNAL CATALOG hudi_catalog_hms
+PROPERTIES
+(
+    "type" = "hudi",
+    "hive.metastore.type" = "hive",
+    "hive.metastore.uris" = "thrift://hive-metastore:9083",
+    "aws.s3.use_instance_profile" = "false",
+    "aws.s3.access_key" = "admin",
+    "aws.s3.secret_key" = "password",
+    "aws.s3.region" = "us-east-1",
+    "aws.s3.enable_ssl" = "false",
+    "aws.s3.enable_path_style_access" = "true",
+    "aws.s3.endpoint" = "http://minio:9000"
+);

ci/SQL/quickstart/hudi/select.sql

@@ -0,0 +1 @@
+SELECT * from hudi_coders_hive\G

ci/SQL/quickstart/hudi/set_catalog.sql

@@ -0,0 +1 @@
+SET CATALOG hudi_catalog_hms;

ci/SQL/quickstart/hudi/show_tables.sql

@@ -0,0 +1 @@
+SHOW tables;

ci/SQL/quickstart/hudi/use_database.sql

@@ -0,0 +1 @@
+USE hudi_sample;

ci/quickstart_hudi_test.go

@@ -0,0 +1,51 @@
+package docs_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("QuickstartHudi", func() {
+
+	When("Running the Hudi Quick Start", Ordered, func() {
+
+		// The database is already initialized, and a connection
+		// is available with the variable `db` which is setup
+		// in the helpers.go file.
+
+		BeforeAll(func() {
+		})
+
+		AfterAll(func() {
+		})
+
+		It("DDL: External Catalog", func() {
+			By("creating an external catalog")
+			SQL := SQLFromFile("SQL/quickstart/hudi/create_catalog.sql")
+			_, err := db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("Setting the catalog")
+			SQL = SQLFromFile("SQL/quickstart/hudi/set_catalog.sql")
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("USEing the database")
+			SQL = SQLFromFile("SQL/quickstart/hudi/use_database.sql")
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("SHOW TABLES")
+			SQL = SQLFromFile("SQL/quickstart/hudi/show_tables.sql")
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("SQL: SELECT FROM the Hudi table", func() {
+
+			By("SELECT")
+			SQL := SQLFromFile("SQL/quickstart/hudi/select.sql")
+			_, err := db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})