Rewrite basic quickstart

README.md

@@ -122,13 +122,20 @@ Code snippets are imported from the tests using [`docusaurus-theme-github-codebl
 ## Running the system
 
 1. Clone this repo
-2. Start StarRocks
+2. Build the Ginkgo container
+
+```bash
+cd ci
+docker build -f ginkgo.Dockerfile -t ginkgo .
+```
+
+3. Start StarRocks
 
 ```bash
 docker compose --profile starrocks up --detach --wait --wait-timeout 60
 ```
 
-3. Run the tests
+4. Run the tests
 
 ```bash
 docker compose run test-harness

ci/SHELL/quickstart/basic/Weather_stream_load

@@ -0,0 +1,13 @@
+#!/bin/ash
+cd /tmp/
+curl --silent --no-buffer                   \
+    --location-trusted -u root:""           \
+    -T ./72505394728.csv                    \
+    -H "label:weather-0"                    \
+    -H "column_separator:,"                 \
+    -H "skip_header:1"                      \
+    -H "enclose:\""                         \
+    -H "max_filter_ratio:1"                 \
+    -H "columns: STATION, DATE, LATITUDE, LONGITUDE, ELEVATION, NAME, REPORT_TYPE, SOURCE, HourlyAltimeterSetting, HourlyDewPointTemperature, HourlyDryBulbTemperature, HourlyPrecipitation, HourlyPresentWeatherType, HourlyPressureChange, HourlyPressureTendency, HourlyRelativeHumidity, HourlySkyConditions, HourlySeaLevelPressure, HourlyStationPressure, HourlyVisibility, HourlyWetBulbTemperature, HourlyWindDirection, HourlyWindGustSpeed, HourlyWindSpeed, Sunrise, Sunset, DailyAverageDewPointTemperature, DailyAverageDryBulbTemperature, DailyAverageRelativeHumidity, DailyAverageSeaLevelPressure, DailyAverageStationPressure, DailyAverageWetBulbTemperature, DailyAverageWindSpeed, DailyCoolingDegreeDays, DailyDepartureFromNormalAverageTemperature, DailyHeatingDegreeDays, DailyMaximumDryBulbTemperature, DailyMinimumDryBulbTemperature, DailyPeakWindDirection, DailyPeakWindSpeed, DailyPrecipitation, DailySnowDepth, DailySnowfall, DailySustainedWindDirection, DailySustainedWindSpeed, DailyWeather, MonthlyAverageRH, MonthlyDaysWithGT001Precip, MonthlyDaysWithGT010Precip, MonthlyDaysWithGT32Temp, MonthlyDaysWithGT90Temp, MonthlyDaysWithLT0Temp, MonthlyDaysWithLT32Temp, MonthlyDepartureFromNormalAverageTemperature, MonthlyDepartureFromNormalCoolingDegreeDays, MonthlyDepartureFromNormalHeatingDegreeDays, MonthlyDepartureFromNormalMaximumTemperature, MonthlyDepartureFromNormalMinimumTemperature, MonthlyDepartureFromNormalPrecipitation, MonthlyDewpointTemperature, MonthlyGreatestPrecip, MonthlyGreatestPrecipDate, MonthlyGreatestSnowDepth, MonthlyGreatestSnowDepthDate, MonthlyGreatestSnowfall, MonthlyGreatestSnowfallDate, MonthlyMaxSeaLevelPressureValue, MonthlyMaxSeaLevelPressureValueDate, MonthlyMaxSeaLevelPressureValueTime, MonthlyMaximumTemperature, MonthlyMeanTemperature, MonthlyMinSeaLevelPressureValue, MonthlyMinSeaLevelPressureValueDate, MonthlyMinSeaLevelPressureValueTime, MonthlyMinimumTemperature, MonthlySeaLevelPressure, MonthlyStationPressure, MonthlyTotalLiquidPrecipitation, MonthlyTotalSnowfall, MonthlyWetBulb, AWND, CDSD, CLDD, DSNW, HDSD, HTDD, NormalsCoolingDegreeDay, NormalsHeatingDegreeDay, ShortDurationEndDate005, ShortDurationEndDate010, ShortDurationEndDate015, ShortDurationEndDate020, ShortDurationEndDate030, ShortDurationEndDate045, ShortDurationEndDate060, ShortDurationEndDate080, ShortDurationEndDate100, ShortDurationEndDate120, ShortDurationEndDate150, ShortDurationEndDate180, ShortDurationPrecipitationValue005, ShortDurationPrecipitationValue010, ShortDurationPrecipitationValue015, ShortDurationPrecipitationValue020, ShortDurationPrecipitationValue030, ShortDurationPrecipitationValue045, ShortDurationPrecipitationValue060, ShortDurationPrecipitationValue080, ShortDurationPrecipitationValue100, ShortDurationPrecipitationValue120, ShortDurationPrecipitationValue150, ShortDurationPrecipitationValue180, REM, BackupDirection, BackupDistance, BackupDistanceUnit, BackupElements, BackupElevation, BackupEquipment, BackupLatitude, BackupLongitude, BackupName, WindEquipmentChangeDate" \
+    -XPUT http://fe:8030/api/quickstart/weatherdata/_stream_load
+

ci/SQL/quickstart/basic/AverageTemp.sql

@@ -0,0 +1,6 @@
+SELECT avg(HourlyDryBulbTemperature),
+       date_trunc("hour", weatherdata.DATE) AS Time
+FROM weatherdata
+GROUP BY Time
+ORDER BY Time ASC
+LIMIT 100;

ci/SQL/quickstart/basic/CrashesPerHour.sql

@@ -0,0 +1,6 @@
+SELECT COUNT(*),
+       date_trunc("hour", crashdata.CRASH_DATE) AS Time
+FROM crashdata
+GROUP BY Time
+ORDER BY Time ASC
+LIMIT 200;

ci/SQL/quickstart/basic/Icy.sql

@@ -0,0 +1,12 @@
+SELECT COUNT(DISTINCT c.COLLISION_ID) AS Crashes,
+       truncate(avg(w.HourlyDryBulbTemperature), 1) AS Temp_F,
+       truncate(avg(w.HourlyVisibility), 2) AS Visibility,
+       max(w.HourlyPrecipitation) AS Precipitation,
+       date_format((date_trunc("hour", c.CRASH_DATE)), '%d %b %Y %H:%i') AS Hour
+FROM crashdata c
+LEFT JOIN weatherdata w
+ON date_trunc("hour", c.CRASH_DATE)=date_trunc("hour", w.DATE)
+WHERE w.HourlyDryBulbTemperature BETWEEN 0.0 AND 40.5 
+GROUP BY Hour
+ORDER BY Crashes DESC
+LIMIT 100;

ci/SQL/quickstart/basic/LowVisibility.sql

@@ -0,0 +1,12 @@
+SELECT COUNT(DISTINCT c.COLLISION_ID) AS Crashes,
+       truncate(avg(w.HourlyDryBulbTemperature), 1) AS Temp_F,
+       truncate(avg(w.HourlyVisibility), 2) AS Visibility,
+       max(w.HourlyPrecipitation) AS Precipitation,
+       date_format((date_trunc("hour", c.CRASH_DATE)), '%d %b %Y %H:%i') AS Hour
+FROM crashdata c
+LEFT JOIN weatherdata w
+ON date_trunc("hour", c.CRASH_DATE)=date_trunc("hour", w.DATE)
+WHERE w.HourlyVisibility BETWEEN 0.0 AND 1.0
+GROUP BY Hour
+ORDER BY Crashes DESC
+LIMIT 100;

ci/docs_test.go

@@ -1,71 +1,73 @@
 package docs_test
 
 import (
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
 	"database/sql"
-	"strings"
-	"os"
 	"fmt"
+	"os"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
 	//"time"
 	"github.com/go-sql-driver/mysql"
 )
 
-
 var _ = Describe("Docs", func() {
 	AWS_S3_ACCESS_KEY := os.Getenv("AWS_S3_ACCESS_KEY")
 	AWS_S3_SECRET_KEY := os.Getenv("AWS_S3_SECRET_KEY")
 
-	When("instance is running in staging", Ordered, func() {
+	When("Loading from S3 docs/loading/s3", Ordered, func() {
 		var db *sql.DB
 
 		cfg := mysql.Config{
-			User:   "root",
-			Passwd: "",
-			Net:    "tcp",
-			Addr:   "fe:9030",
+			User:                 "root",
+			Passwd:               "",
+			Net:                  "tcp",
+			Addr:                 "fe:9030",
 			AllowNativePasswords: true,
 		}
 
 		BeforeAll(func() {
+			By("Connecting to StarRocks FE")
 			db, _ = sql.Open("mysql", cfg.FormatDSN())
 			db.SetMaxOpenConns(1)
+			Expect(db.Ping()).Should(Succeed())
 		})
 
 		AfterAll(func() {
 			var err error
+			By("DROP table user_behavior_inferred")
 			_, err = db.Exec(`DROP TABLE IF EXISTS DocsQA.user_behavior_inferred`)
 			Expect(err).ToNot(HaveOccurred())
+
+			By("DROP DB DocsQA")
 			_, err = db.Exec(`DROP DATABASE IF EXISTS DocsQA`)
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should be able to run SQL commands", func() {
+		It("DDL: setup DocsQA DB", func() {
 			By("creating a database")
 			_, err := db.Exec(`CREATE DATABASE IF NOT EXISTS DocsQA`)
 			Expect(err).ToNot(HaveOccurred())
-		})
 
-		It("should be able to run SQL commands", func() {
 			By("choosing a database")
-			_, err := db.Exec(`USE DocsQA`)
+			_, err = db.Exec(`USE DocsQA`)
 			Expect(err).ToNot(HaveOccurred())
-		})
 
-		It("should be able to run SQL commands", func() {
 			By("setting the number of replicas")
-			_, err := db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`)
+			_, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`)
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should be able to run SQL commands", func() {
+		It("Use the FILES table fxn", func() {
 			By("creating and populating a table")
 			b, err := os.ReadFile("SQL/files_table_fxn.sql")
 			if err != nil {
 				fmt.Print(err)
 			}
 			SQL := string(b)
-			re := strings.NewReplacer( "AAAAAAAAAAAAAAAAAAAA", AWS_S3_ACCESS_KEY, "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", AWS_S3_SECRET_KEY)
+			re := strings.NewReplacer("AAAAAAAAAAAAAAAAAAAA", AWS_S3_ACCESS_KEY, "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", AWS_S3_SECRET_KEY)
 
 			SQLWithKey := re.Replace(SQL)
 			_, err = db.Exec(SQLWithKey)

ci/quickstart_basic_test.go

@@ -18,7 +18,7 @@ var _ = Describe("QuickstartBasic", func() {
 	//AWS_S3_ACCESS_KEY := os.Getenv("AWS_S3_ACCESS_KEY")
 	//AWS_S3_SECRET_KEY := os.Getenv("AWS_S3_SECRET_KEY")
 
-	When("Running a single FE and BE via Docker compose", Ordered, func() {
+	When("Running the basic Quick Start", Ordered, func() {
 		var db *sql.DB
 
 		cfg := mysql.Config{
@@ -32,78 +32,121 @@ var _ = Describe("QuickstartBasic", func() {
 		BeforeAll(func() {
 			// download the crash data in /tmp/ dir
 			// https://stackoverflow.com/questions/16703647/why-does-curl-return-error-23-failed-writing-body
+			By("Downloading the NYPD Crash data")
 			NYPDCurl := exec.Command("/usr/bin/curl","-s","-N","-o","/tmp/NYPD_Crash_Data.csv","https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/datasets/NYPD_Crash_Data.csv")
 			err := NYPDCurl.Start()
 			Expect(err).ToNot(HaveOccurred())
 			err = NYPDCurl.Wait()
 			Expect(err).ToNot(HaveOccurred())
 
+			By("Downloading the NOAA weather data")
 			WeatherCurl := exec.Command("/usr/bin/curl","-s","-N","-o","/tmp/72505394728.csv","https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/datasets/72505394728.csv")
 			err = WeatherCurl.Start()
 			Expect(err).ToNot(HaveOccurred())
 			err = WeatherCurl.Wait()
 			Expect(err).ToNot(HaveOccurred())
 
-
 			// Connect to the database
-			db, _ = sql.Open("mysql", cfg.FormatDSN())
+			By("Connecting to StarRocks FE")
+			db, err = sql.Open("mysql", cfg.FormatDSN())
 			db.SetMaxOpenConns(1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(db.Ping()).Should(Succeed())
 
 		})
 
-		//AfterAll(func() {
-			//_, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`)
-			//Expect(err).ToNot(HaveOccurred())
-		//})
+		AfterAll(func() {
+			By("dropping quickstart DB")
+			_, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`)
+			Expect(err).ToNot(HaveOccurred())
+		})
 
-		It("should be able to run SQL commands", func() {
+		It("DDL: Setup quickstart DB", func() {
 			By("creating a database")
 			_, err := db.Exec(`CREATE DATABASE IF NOT EXISTS quickstart`)
 			Expect(err).ToNot(HaveOccurred())
-		})
 
-		It("should be able to run SQL commands", func() {
 			By("choosing a database")
-			_, err := db.Exec(`USE quickstart`)
+			_, err = db.Exec(`USE quickstart`)
 			Expect(err).ToNot(HaveOccurred())
-		})
 
-		It("should be able to run SQL commands", func() {
 			By("setting the number of replicas")
-			_, err := db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`)
+			_, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`)
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should be able to run SQL commands", func() {
+		It("DDL: Create quickstart tables", func() {
 			By("creating the crash data table")
-			b, err := os.ReadFile("SQL/NYPD_table.sql")
+			b, err := os.ReadFile("SQL/quickstart/basic/NYPD_table.sql")
 			if err != nil {
 				fmt.Print(err)
 			}
 			SQL := string(b)
 			_, err = db.Exec(SQL)
 			Expect(err).ToNot(HaveOccurred())
-		})
 
-		It("should be able to run SQL commands", func() {
 			By("creating the weather data table")
-			b, err := os.ReadFile("SQL/Weather_table.sql")
+			b, err = os.ReadFile("SQL/quickstart/basic/Weather_table.sql")
 			if err != nil {
 				fmt.Print(err)
 			}
-			SQL := string(b)
+			SQL = string(b)
 			_, err = db.Exec(SQL)
 			Expect(err).ToNot(HaveOccurred())
 		})
 
 		It("should be able to load data via stream load", func() {
 			By("uploading the NYPD crash data")
-			NYPDStreamLoad := exec.Command("SHELL/NYPD_stream_load")
+			NYPDStreamLoad := exec.Command("SHELL/quickstart/basic/NYPD_stream_load")
 			err := NYPDStreamLoad.Start()
 			Expect(err).ToNot(HaveOccurred())
 			err = NYPDStreamLoad.Wait()
 			Expect(err).ToNot(HaveOccurred())
+
+			By("uploading the NOAA weather data")
+			WeatherStreamLoad := exec.Command("SHELL/quickstart/basic/Weather_stream_load")
+			err = WeatherStreamLoad.Start()
+			Expect(err).ToNot(HaveOccurred())
+			err = WeatherStreamLoad.Wait()
+			Expect(err).ToNot(HaveOccurred())
 		})
 
+		It("should be able to query tables", func() {
+			By("querying the crash data table")
+			b, err := os.ReadFile("SQL/quickstart/basic/CrashesPerHour.sql")
+			if err != nil {
+				fmt.Print(err)
+			}
+			SQL := string(b)
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("querying the weather data table")
+			b, err = os.ReadFile("SQL/quickstart/basic/AverageTemp.sql")
+			if err != nil {
+				fmt.Print(err)
+			}
+			SQL = string(b)
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("JOINing to see impact of low visibility")
+			b, err = os.ReadFile("SQL/quickstart/basic/LowVisibility.sql")
+			if err != nil {
+				fmt.Print(err)
+			}
+			SQL = string(b)
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("JOINing to see impact of icy weather")
+			b, err = os.ReadFile("SQL/quickstart/basic/Icy.sql")
+			if err != nil {
+				fmt.Print(err)
+			}
+			SQL = string(b)
+			_, err = db.Exec(SQL)
+			Expect(err).ToNot(HaveOccurred())
+		})
 	})
 })

docker-compose.yml

@@ -57,10 +57,6 @@ services:
     build:
       context: ci
       dockerfile: ginkgo.Dockerfile
-        #depends_on:
-        #starrocks-be:
-        #condition: service_healthy
-        #restart: true
     command:
       ginkgo -vv