diff --git a/README.md b/README.md index 2a17056..5d9c05f 100644 --- a/README.md +++ b/README.md @@ -122,13 +122,20 @@ Code snippets are imported from the tests using [`docusaurus-theme-github-codebl ## Running the system 1. Clone this repo -2. Start StarRocks +2. Build the Ginkgo container + +```bash +cd ci +docker build -f ginkgo.Dockerfile -t ginkgo . +``` + +3. Start StarRocks ```bash docker compose --profile starrocks up --detach --wait --wait-timeout 60 ``` -3. Run the tests +4. Run the tests ```bash docker compose run test-harness diff --git a/ci/SHELL/NYPD_stream_load b/ci/SHELL/quickstart/basic/NYPD_stream_load similarity index 100% rename from ci/SHELL/NYPD_stream_load rename to ci/SHELL/quickstart/basic/NYPD_stream_load diff --git a/ci/SHELL/quickstart/basic/Weather_stream_load b/ci/SHELL/quickstart/basic/Weather_stream_load new file mode 100755 index 0000000..ba94e53 --- /dev/null +++ b/ci/SHELL/quickstart/basic/Weather_stream_load @@ -0,0 +1,13 @@ +#!/bin/ash +cd /tmp/ +curl --silent --no-buffer \ + --location-trusted -u root:"" \ + -T ./72505394728.csv \ + -H "label:weather-0" \ + -H "column_separator:," \ + -H "skip_header:1" \ + -H "enclose:\"" \ + -H "max_filter_ratio:1" \ + -H "columns: STATION, DATE, LATITUDE, LONGITUDE, ELEVATION, NAME, REPORT_TYPE, SOURCE, HourlyAltimeterSetting, HourlyDewPointTemperature, HourlyDryBulbTemperature, HourlyPrecipitation, HourlyPresentWeatherType, HourlyPressureChange, HourlyPressureTendency, HourlyRelativeHumidity, HourlySkyConditions, HourlySeaLevelPressure, HourlyStationPressure, HourlyVisibility, HourlyWetBulbTemperature, HourlyWindDirection, HourlyWindGustSpeed, HourlyWindSpeed, Sunrise, Sunset, DailyAverageDewPointTemperature, DailyAverageDryBulbTemperature, DailyAverageRelativeHumidity, DailyAverageSeaLevelPressure, DailyAverageStationPressure, DailyAverageWetBulbTemperature, DailyAverageWindSpeed, DailyCoolingDegreeDays, DailyDepartureFromNormalAverageTemperature, DailyHeatingDegreeDays, DailyMaximumDryBulbTemperature, DailyMinimumDryBulbTemperature, DailyPeakWindDirection, DailyPeakWindSpeed, DailyPrecipitation, DailySnowDepth, DailySnowfall, DailySustainedWindDirection, DailySustainedWindSpeed, DailyWeather, MonthlyAverageRH, MonthlyDaysWithGT001Precip, MonthlyDaysWithGT010Precip, MonthlyDaysWithGT32Temp, MonthlyDaysWithGT90Temp, MonthlyDaysWithLT0Temp, MonthlyDaysWithLT32Temp, MonthlyDepartureFromNormalAverageTemperature, MonthlyDepartureFromNormalCoolingDegreeDays, MonthlyDepartureFromNormalHeatingDegreeDays, MonthlyDepartureFromNormalMaximumTemperature, MonthlyDepartureFromNormalMinimumTemperature, MonthlyDepartureFromNormalPrecipitation, MonthlyDewpointTemperature, MonthlyGreatestPrecip, MonthlyGreatestPrecipDate, MonthlyGreatestSnowDepth, MonthlyGreatestSnowDepthDate, MonthlyGreatestSnowfall, MonthlyGreatestSnowfallDate, MonthlyMaxSeaLevelPressureValue, MonthlyMaxSeaLevelPressureValueDate, MonthlyMaxSeaLevelPressureValueTime, MonthlyMaximumTemperature, MonthlyMeanTemperature, MonthlyMinSeaLevelPressureValue, MonthlyMinSeaLevelPressureValueDate, MonthlyMinSeaLevelPressureValueTime, MonthlyMinimumTemperature, MonthlySeaLevelPressure, MonthlyStationPressure, MonthlyTotalLiquidPrecipitation, MonthlyTotalSnowfall, MonthlyWetBulb, AWND, CDSD, CLDD, DSNW, HDSD, HTDD, NormalsCoolingDegreeDay, NormalsHeatingDegreeDay, ShortDurationEndDate005, ShortDurationEndDate010, ShortDurationEndDate015, ShortDurationEndDate020, ShortDurationEndDate030, ShortDurationEndDate045, ShortDurationEndDate060, ShortDurationEndDate080, ShortDurationEndDate100, ShortDurationEndDate120, ShortDurationEndDate150, ShortDurationEndDate180, ShortDurationPrecipitationValue005, ShortDurationPrecipitationValue010, ShortDurationPrecipitationValue015, ShortDurationPrecipitationValue020, ShortDurationPrecipitationValue030, ShortDurationPrecipitationValue045, ShortDurationPrecipitationValue060, ShortDurationPrecipitationValue080, ShortDurationPrecipitationValue100, ShortDurationPrecipitationValue120, ShortDurationPrecipitationValue150, ShortDurationPrecipitationValue180, REM, BackupDirection, BackupDistance, BackupDistanceUnit, BackupElements, BackupElevation, BackupEquipment, BackupLatitude, BackupLongitude, BackupName, WindEquipmentChangeDate" \ + -XPUT http://fe:8030/api/quickstart/weatherdata/_stream_load + diff --git a/ci/SQL/quickstart/basic/AverageTemp.sql b/ci/SQL/quickstart/basic/AverageTemp.sql new file mode 100644 index 0000000..f5d1a79 --- /dev/null +++ b/ci/SQL/quickstart/basic/AverageTemp.sql @@ -0,0 +1,6 @@ +SELECT avg(HourlyDryBulbTemperature), + date_trunc("hour", weatherdata.DATE) AS Time +FROM weatherdata +GROUP BY Time +ORDER BY Time ASC +LIMIT 100; diff --git a/ci/SQL/quickstart/basic/CrashesPerHour.sql b/ci/SQL/quickstart/basic/CrashesPerHour.sql new file mode 100644 index 0000000..c481fec --- /dev/null +++ b/ci/SQL/quickstart/basic/CrashesPerHour.sql @@ -0,0 +1,6 @@ +SELECT COUNT(*), + date_trunc("hour", crashdata.CRASH_DATE) AS Time +FROM crashdata +GROUP BY Time +ORDER BY Time ASC +LIMIT 200; diff --git a/ci/SQL/quickstart/basic/Icy.sql b/ci/SQL/quickstart/basic/Icy.sql new file mode 100644 index 0000000..cea2553 --- /dev/null +++ b/ci/SQL/quickstart/basic/Icy.sql @@ -0,0 +1,12 @@ +SELECT COUNT(DISTINCT c.COLLISION_ID) AS Crashes, + truncate(avg(w.HourlyDryBulbTemperature), 1) AS Temp_F, + truncate(avg(w.HourlyVisibility), 2) AS Visibility, + max(w.HourlyPrecipitation) AS Precipitation, + date_format((date_trunc("hour", c.CRASH_DATE)), '%d %b %Y %H:%i') AS Hour +FROM crashdata c +LEFT JOIN weatherdata w +ON date_trunc("hour", c.CRASH_DATE)=date_trunc("hour", w.DATE) +WHERE w.HourlyDryBulbTemperature BETWEEN 0.0 AND 40.5 +GROUP BY Hour +ORDER BY Crashes DESC +LIMIT 100; diff --git a/ci/SQL/quickstart/basic/LowVisibility.sql b/ci/SQL/quickstart/basic/LowVisibility.sql new file mode 100644 index 0000000..c0ed93f --- /dev/null +++ b/ci/SQL/quickstart/basic/LowVisibility.sql @@ -0,0 +1,12 @@ +SELECT COUNT(DISTINCT c.COLLISION_ID) AS Crashes, + truncate(avg(w.HourlyDryBulbTemperature), 1) AS Temp_F, + truncate(avg(w.HourlyVisibility), 2) AS Visibility, + max(w.HourlyPrecipitation) AS Precipitation, + date_format((date_trunc("hour", c.CRASH_DATE)), '%d %b %Y %H:%i') AS Hour +FROM crashdata c +LEFT JOIN weatherdata w +ON date_trunc("hour", c.CRASH_DATE)=date_trunc("hour", w.DATE) +WHERE w.HourlyVisibility BETWEEN 0.0 AND 1.0 +GROUP BY Hour +ORDER BY Crashes DESC +LIMIT 100; diff --git a/ci/SQL/NYPD_table.sql b/ci/SQL/quickstart/basic/NYPD_table.sql similarity index 100% rename from ci/SQL/NYPD_table.sql rename to ci/SQL/quickstart/basic/NYPD_table.sql diff --git a/ci/SQL/Weather_table.sql b/ci/SQL/quickstart/basic/Weather_table.sql similarity index 100% rename from ci/SQL/Weather_table.sql rename to ci/SQL/quickstart/basic/Weather_table.sql diff --git a/ci/docs_test.go b/ci/docs_test.go index aeaa043..e02701d 100644 --- a/ci/docs_test.go +++ b/ci/docs_test.go @@ -1,71 +1,73 @@ package docs_test import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "database/sql" - "strings" - "os" "fmt" + "os" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + //"time" "github.com/go-sql-driver/mysql" ) - var _ = Describe("Docs", func() { AWS_S3_ACCESS_KEY := os.Getenv("AWS_S3_ACCESS_KEY") AWS_S3_SECRET_KEY := os.Getenv("AWS_S3_SECRET_KEY") - When("instance is running in staging", Ordered, func() { + When("Loading from S3 docs/loading/s3", Ordered, func() { var db *sql.DB cfg := mysql.Config{ - User: "root", - Passwd: "", - Net: "tcp", - Addr: "fe:9030", + User: "root", + Passwd: "", + Net: "tcp", + Addr: "fe:9030", AllowNativePasswords: true, } BeforeAll(func() { + By("Connecting to StarRocks FE") db, _ = sql.Open("mysql", cfg.FormatDSN()) db.SetMaxOpenConns(1) + Expect(db.Ping()).Should(Succeed()) }) AfterAll(func() { var err error + By("DROP table user_behavior_inferred") _, err = db.Exec(`DROP TABLE IF EXISTS DocsQA.user_behavior_inferred`) Expect(err).ToNot(HaveOccurred()) + + By("DROP DB DocsQA") _, err = db.Exec(`DROP DATABASE IF EXISTS DocsQA`) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to run SQL commands", func() { + It("DDL: setup DocsQA DB", func() { By("creating a database") _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS DocsQA`) Expect(err).ToNot(HaveOccurred()) - }) - It("should be able to run SQL commands", func() { By("choosing a database") - _, err := db.Exec(`USE DocsQA`) + _, err = db.Exec(`USE DocsQA`) Expect(err).ToNot(HaveOccurred()) - }) - It("should be able to run SQL commands", func() { By("setting the number of replicas") - _, err := db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`) + _, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to run SQL commands", func() { + It("Use the FILES table fxn", func() { By("creating and populating a table") b, err := os.ReadFile("SQL/files_table_fxn.sql") if err != nil { fmt.Print(err) } SQL := string(b) - re := strings.NewReplacer( "AAAAAAAAAAAAAAAAAAAA", AWS_S3_ACCESS_KEY, "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", AWS_S3_SECRET_KEY) + re := strings.NewReplacer("AAAAAAAAAAAAAAAAAAAA", AWS_S3_ACCESS_KEY, "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", AWS_S3_SECRET_KEY) SQLWithKey := re.Replace(SQL) _, err = db.Exec(SQLWithKey) diff --git a/ci/quickstart_basic_test.go b/ci/quickstart_basic_test.go index 97b1ba6..2aeb188 100644 --- a/ci/quickstart_basic_test.go +++ b/ci/quickstart_basic_test.go @@ -18,7 +18,7 @@ var _ = Describe("QuickstartBasic", func() { //AWS_S3_ACCESS_KEY := os.Getenv("AWS_S3_ACCESS_KEY") //AWS_S3_SECRET_KEY := os.Getenv("AWS_S3_SECRET_KEY") - When("Running a single FE and BE via Docker compose", Ordered, func() { + When("Running the basic Quick Start", Ordered, func() { var db *sql.DB cfg := mysql.Config{ @@ -32,78 +32,121 @@ var _ = Describe("QuickstartBasic", func() { BeforeAll(func() { // download the crash data in /tmp/ dir // https://stackoverflow.com/questions/16703647/why-does-curl-return-error-23-failed-writing-body + By("Downloading the NYPD Crash data") NYPDCurl := exec.Command("/usr/bin/curl","-s","-N","-o","/tmp/NYPD_Crash_Data.csv","https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/datasets/NYPD_Crash_Data.csv") err := NYPDCurl.Start() Expect(err).ToNot(HaveOccurred()) err = NYPDCurl.Wait() Expect(err).ToNot(HaveOccurred()) + By("Downloading the NOAA weather data") WeatherCurl := exec.Command("/usr/bin/curl","-s","-N","-o","/tmp/72505394728.csv","https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/datasets/72505394728.csv") err = WeatherCurl.Start() Expect(err).ToNot(HaveOccurred()) err = WeatherCurl.Wait() Expect(err).ToNot(HaveOccurred()) - // Connect to the database - db, _ = sql.Open("mysql", cfg.FormatDSN()) + By("Connecting to StarRocks FE") + db, err = sql.Open("mysql", cfg.FormatDSN()) db.SetMaxOpenConns(1) + Expect(err).ToNot(HaveOccurred()) + Expect(db.Ping()).Should(Succeed()) }) - //AfterAll(func() { - //_, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`) - //Expect(err).ToNot(HaveOccurred()) - //}) + AfterAll(func() { + By("dropping quickstart DB") + _, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`) + Expect(err).ToNot(HaveOccurred()) + }) - It("should be able to run SQL commands", func() { + It("DDL: Setup quickstart DB", func() { By("creating a database") _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS quickstart`) Expect(err).ToNot(HaveOccurred()) - }) - It("should be able to run SQL commands", func() { By("choosing a database") - _, err := db.Exec(`USE quickstart`) + _, err = db.Exec(`USE quickstart`) Expect(err).ToNot(HaveOccurred()) - }) - It("should be able to run SQL commands", func() { By("setting the number of replicas") - _, err := db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`) + _, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "1");`) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to run SQL commands", func() { + It("DDL: Create quickstart tables", func() { By("creating the crash data table") - b, err := os.ReadFile("SQL/NYPD_table.sql") + b, err := os.ReadFile("SQL/quickstart/basic/NYPD_table.sql") if err != nil { fmt.Print(err) } SQL := string(b) _, err = db.Exec(SQL) Expect(err).ToNot(HaveOccurred()) - }) - It("should be able to run SQL commands", func() { By("creating the weather data table") - b, err := os.ReadFile("SQL/Weather_table.sql") + b, err = os.ReadFile("SQL/quickstart/basic/Weather_table.sql") if err != nil { fmt.Print(err) } - SQL := string(b) + SQL = string(b) _, err = db.Exec(SQL) Expect(err).ToNot(HaveOccurred()) }) It("should be able to load data via stream load", func() { By("uploading the NYPD crash data") - NYPDStreamLoad := exec.Command("SHELL/NYPD_stream_load") + NYPDStreamLoad := exec.Command("SHELL/quickstart/basic/NYPD_stream_load") err := NYPDStreamLoad.Start() Expect(err).ToNot(HaveOccurred()) err = NYPDStreamLoad.Wait() Expect(err).ToNot(HaveOccurred()) + + By("uploading the NOAA weather data") + WeatherStreamLoad := exec.Command("SHELL/quickstart/basic/Weather_stream_load") + err = WeatherStreamLoad.Start() + Expect(err).ToNot(HaveOccurred()) + err = WeatherStreamLoad.Wait() + Expect(err).ToNot(HaveOccurred()) }) + It("should be able to query tables", func() { + By("querying the crash data table") + b, err := os.ReadFile("SQL/quickstart/basic/CrashesPerHour.sql") + if err != nil { + fmt.Print(err) + } + SQL := string(b) + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("querying the weather data table") + b, err = os.ReadFile("SQL/quickstart/basic/AverageTemp.sql") + if err != nil { + fmt.Print(err) + } + SQL = string(b) + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("JOINing to see impact of low visibility") + b, err = os.ReadFile("SQL/quickstart/basic/LowVisibility.sql") + if err != nil { + fmt.Print(err) + } + SQL = string(b) + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("JOINing to see impact of icy weather") + b, err = os.ReadFile("SQL/quickstart/basic/Icy.sql") + if err != nil { + fmt.Print(err) + } + SQL = string(b) + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + }) }) }) diff --git a/docker-compose.yml b/docker-compose.yml index 516d13b..14fe5f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,10 +57,6 @@ services: build: context: ci dockerfile: ginkgo.Dockerfile - #depends_on: - #starrocks-be: - #condition: service_healthy - #restart: true command: ginkgo -vv