Skip to content

Commit

Permalink
Merge pull request #24 from DanRoscigno/shared_data
Browse files Browse the repository at this point in the history
Add shared data tests
DanRoscigno authored Feb 13, 2024
2 parents c493035 + a7f7de8 commit 4de8c6e
Showing 12 changed files with 207 additions and 23 deletions.
1 change: 1 addition & 0 deletions .github/workflows/test_with_allin1.yml
Original file line number Diff line number Diff line change
@@ -59,6 +59,7 @@ jobs:
env:
AWS_S3_ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY }}
AWS_S3_SECRET_KEY: ${{ secrets.AWS_S3_SECRET_KEY }}
SR_FE_HOST: 'localhost'
run: ginkgo -v --focus-file=./quickstart_basic_test.go
working-directory: ./ci

28 changes: 15 additions & 13 deletions .github/workflows/test_with_oneFE_oneBE.yml
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
name: Test the docs

on:
schedule:
- cron: "5 9 * * 1"
push:
branches: [ main ]
paths:
- 'ci/**/*'
pull_request:
branches: [ main ]
paths:
- 'ci/**/*'
- '.github/workflows/test_the_docs.yml'
- 'docker-compose.yml'
on:
workflow_dispatch:
#on:
#schedule:
#- cron: "5 9 * * 1"
#push:
#branches: [ main ]
#paths:
#- 'ci/**/*'
#pull_request:
#branches: [ main ]
#paths:
#- 'ci/**/*'
#- '.github/workflows/test_the_docs.yml'
#- 'docker-compose.yml'

jobs:
build:
58 changes: 58 additions & 0 deletions .github/workflows/test_with_shared_data.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
name: Test with Shared Data

on:
schedule:
- cron: "5 9 * * 1"
push:
branches: [ main ]
paths:
- 'ci/**/quickstart/shared_data/*'
- '.github/workflows/test_with_shared_data.yml'
- 'quickstart_shared_data_test.go'
- 'helper.go'
pull_request:
branches: [ main ]
paths:
- 'ci/**/quickstart/shared_data/*'
- '.github/workflows/test_with_shared_data.yml'
- 'quickstart_shared_data_test.go'
- 'helper.go'

jobs:
build:

name: Build and test
runs-on: ubuntu-latest

steps:
# Checkout the repo as this CI needs:
# - the compose file for StarRocks and Ginkgo/Gomega
- name: Checkout Test repo
uses: actions/checkout@v4
#with:
#path: testing

# Build the test harness container (Golnag/Ginkgo/Gomega)
- name: Build Ginkgo container
run: |
curl -o shared_data-compose.yml \
https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/docker-compose.yml
docker compose -f shared_data-compose.yml -f test-harness-docker-compose.yml build
- name: Start StarRocks and MinIO
run: |
docker compose \
-f shared_data-compose.yml \
-f test-harness-docker-compose.yml \
up --detach --wait --wait-timeout 100
# Any tests that will run against the StarRocks env would be
# launched in steps like this one. Make sure to reset the
# StarRocks environment after each run (remove any tables
# and databases created, and reset any settings to the default)
#
# The ginkgo command uses `--focus-file` to run only the one test
# file.
- name: Test; Shared Data SQL test
if: always()
run: docker compose -f shared_data-compose.yml -f test-harness-docker-compose.yml exec -e SR_FE_HOST='starrocks-fe' test-harness ginkgo -v --focus-file=./quickstart_shared_data_test.go
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
.env
shared_data-compose.yml
2 changes: 1 addition & 1 deletion ci/SHELL/quickstart/basic/NYPD_download
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
cd /tmp/
curl --silent --no-buffer \
-o /tmp/NYPD_Crash_Data.csv \
4 changes: 2 additions & 2 deletions ci/SHELL/quickstart/basic/NYPD_stream_load
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
cd /tmp/
curl --silent --no-buffer \
--location-trusted -u root:"" \
@@ -9,4 +9,4 @@ curl --silent --no-buffer \
-H "enclose:\"" \
-H "max_filter_ratio:1" \
-H "columns:tmp_CRASH_DATE, tmp_CRASH_TIME, CRASH_DATE=str_to_date(concat_ws(' ', tmp_CRASH_DATE, tmp_CRASH_TIME), '%m/%d/%Y %H:%i'),BOROUGH,ZIP_CODE,LATITUDE,LONGITUDE,LOCATION,ON_STREET_NAME,CROSS_STREET_NAME,OFF_STREET_NAME,NUMBER_OF_PERSONS_INJURED,NUMBER_OF_PERSONS_KILLED,NUMBER_OF_PEDESTRIANS_INJURED,NUMBER_OF_PEDESTRIANS_KILLED,NUMBER_OF_CYCLIST_INJURED,NUMBER_OF_CYCLIST_KILLED,NUMBER_OF_MOTORIST_INJURED,NUMBER_OF_MOTORIST_KILLED,CONTRIBUTING_FACTOR_VEHICLE_1,CONTRIBUTING_FACTOR_VEHICLE_2,CONTRIBUTING_FACTOR_VEHICLE_3,CONTRIBUTING_FACTOR_VEHICLE_4,CONTRIBUTING_FACTOR_VEHICLE_5,COLLISION_ID,VEHICLE_TYPE_CODE_1,VEHICLE_TYPE_CODE_2,VEHICLE_TYPE_CODE_3,VEHICLE_TYPE_CODE_4,VEHICLE_TYPE_CODE_5" \
-XPUT http://localhost:8030/api/quickstart/crashdata/_stream_load
-XPUT http://${SR_FE_HOST}:8030/api/quickstart/crashdata/_stream_load
2 changes: 1 addition & 1 deletion ci/SHELL/quickstart/basic/Weather_download
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
cd /tmp/
curl --silent --no-buffer \
-o 72505394728.csv \
4 changes: 2 additions & 2 deletions ci/SHELL/quickstart/basic/Weather_stream_load
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/sh
cd /tmp/
curl --silent --no-buffer \
--location-trusted -u root:"" \
@@ -9,5 +9,5 @@ curl --silent --no-buffer \
-H "enclose:\"" \
-H "max_filter_ratio:1" \
-H "columns: STATION, DATE, LATITUDE, LONGITUDE, ELEVATION, NAME, REPORT_TYPE, SOURCE, HourlyAltimeterSetting, HourlyDewPointTemperature, HourlyDryBulbTemperature, HourlyPrecipitation, HourlyPresentWeatherType, HourlyPressureChange, HourlyPressureTendency, HourlyRelativeHumidity, HourlySkyConditions, HourlySeaLevelPressure, HourlyStationPressure, HourlyVisibility, HourlyWetBulbTemperature, HourlyWindDirection, HourlyWindGustSpeed, HourlyWindSpeed, Sunrise, Sunset, DailyAverageDewPointTemperature, DailyAverageDryBulbTemperature, DailyAverageRelativeHumidity, DailyAverageSeaLevelPressure, DailyAverageStationPressure, DailyAverageWetBulbTemperature, DailyAverageWindSpeed, DailyCoolingDegreeDays, DailyDepartureFromNormalAverageTemperature, DailyHeatingDegreeDays, DailyMaximumDryBulbTemperature, DailyMinimumDryBulbTemperature, DailyPeakWindDirection, DailyPeakWindSpeed, DailyPrecipitation, DailySnowDepth, DailySnowfall, DailySustainedWindDirection, DailySustainedWindSpeed, DailyWeather, MonthlyAverageRH, MonthlyDaysWithGT001Precip, MonthlyDaysWithGT010Precip, MonthlyDaysWithGT32Temp, MonthlyDaysWithGT90Temp, MonthlyDaysWithLT0Temp, MonthlyDaysWithLT32Temp, MonthlyDepartureFromNormalAverageTemperature, MonthlyDepartureFromNormalCoolingDegreeDays, MonthlyDepartureFromNormalHeatingDegreeDays, MonthlyDepartureFromNormalMaximumTemperature, MonthlyDepartureFromNormalMinimumTemperature, MonthlyDepartureFromNormalPrecipitation, MonthlyDewpointTemperature, MonthlyGreatestPrecip, MonthlyGreatestPrecipDate, MonthlyGreatestSnowDepth, MonthlyGreatestSnowDepthDate, MonthlyGreatestSnowfall, MonthlyGreatestSnowfallDate, MonthlyMaxSeaLevelPressureValue, MonthlyMaxSeaLevelPressureValueDate, MonthlyMaxSeaLevelPressureValueTime, MonthlyMaximumTemperature, MonthlyMeanTemperature, MonthlyMinSeaLevelPressureValue, MonthlyMinSeaLevelPressureValueDate, MonthlyMinSeaLevelPressureValueTime, MonthlyMinimumTemperature, MonthlySeaLevelPressure, MonthlyStationPressure, MonthlyTotalLiquidPrecipitation, MonthlyTotalSnowfall, MonthlyWetBulb, AWND, CDSD, CLDD, DSNW, HDSD, HTDD, NormalsCoolingDegreeDay, NormalsHeatingDegreeDay, ShortDurationEndDate005, ShortDurationEndDate010, ShortDurationEndDate015, ShortDurationEndDate020, ShortDurationEndDate030, ShortDurationEndDate045, ShortDurationEndDate060, ShortDurationEndDate080, ShortDurationEndDate100, ShortDurationEndDate120, ShortDurationEndDate150, ShortDurationEndDate180, ShortDurationPrecipitationValue005, ShortDurationPrecipitationValue010, ShortDurationPrecipitationValue015, ShortDurationPrecipitationValue020, ShortDurationPrecipitationValue030, ShortDurationPrecipitationValue045, ShortDurationPrecipitationValue060, ShortDurationPrecipitationValue080, ShortDurationPrecipitationValue100, ShortDurationPrecipitationValue120, ShortDurationPrecipitationValue150, ShortDurationPrecipitationValue180, REM, BackupDirection, BackupDistance, BackupDistanceUnit, BackupElements, BackupElevation, BackupEquipment, BackupLatitude, BackupLongitude, BackupName, WindEquipmentChangeDate" \
-XPUT http://localhost:8030/api/quickstart/weatherdata/_stream_load
-XPUT http://${SR_FE_HOST}:8030/api/quickstart/weatherdata/_stream_load

7 changes: 7 additions & 0 deletions ci/SQL/quickstart/shared_data/shared_data_DDL.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
SHOW STORAGE VOLUMES;

DESC STORAGE VOLUME builtin_storage_volume;

CREATE DATABASE IF NOT EXISTS quickstart;

USE quickstart;
4 changes: 3 additions & 1 deletion ci/helper.go
Original file line number Diff line number Diff line change
@@ -12,11 +12,13 @@ import (
var db *sql.DB

func GetDSNConnection() (*sql.DB, error) {
SR_FE_HOST := os.Getenv("SR_FE_HOST")
fmt.Print("SR HOST is " + SR_FE_HOST)
cfg := mysql.Config{
User: "root",
Passwd: "",
Net: "tcp",
Addr: "localhost:9030",
Addr: SR_FE_HOST + ":9030",
AllowNativePasswords: true,
}
return sql.Open("mysql", cfg.FormatDSN())
94 changes: 94 additions & 0 deletions ci/quickstart_shared_data_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package docs_test

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("QuickstartSharedData", func() {

When("Running the SharedData Quick Start", Ordered, func() {

// The database is already initialized, and a connection
// is available with the variable `db` which is setup
// in the helpers.go file.

BeforeAll(func() {
// download the crash data in /tmp/ dir
// https://stackoverflow.com/questions/16703647/why-does-curl-return-error-23-failed-writing-body
By("Downloading the NYPD Crash data")
LongRunningScript("SHELL/quickstart/basic/NYPD_download")

By("Downloading the NOAA weather data")
LongRunningScript("SHELL/quickstart/basic/Weather_download")
})

AfterAll(func() {
By("dropping quickstart DB")
_, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`)
Expect(err).ToNot(HaveOccurred())

By("Reset settings")
_, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "3");`)
Expect(err).ToNot(HaveOccurred())
})

It("DDL: Setup quickstart DB", func() {
By("creating a database")
SQL := SQLFromFile("SQL/quickstart/shared_data/shared_data_DDL.sql")
_, err := db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())
})

// Note: The rest of these tests use the same
// queries as the basic quick start, so they are
// sourced from the basic directory.

It("DDL: Create quickstart tables", func() {
By("creating the crash data table")
SQL := SQLFromFile("SQL/quickstart/basic/NYPD_table.sql")
_, err := db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())
})

It("DDL: Create quickstart tables", func() {

By("creating the weather data table")
SQL := SQLFromFile("SQL/quickstart/basic/Weather_table.sql")
_, err := db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())
})

It("should be able to load data via stream load", func() {
By("uploading the NYPD crash data")
LongRunningScript("SHELL/quickstart/basic/NYPD_stream_load")
})
It("should be able to load data via stream load", func() {

By("uploading the NOAA weather data")
LongRunningScript("SHELL/quickstart/basic/Weather_stream_load")
})

It("should be able to query tables", func() {
By("querying the crash data table")
SQL := SQLFromFile("SQL/quickstart/basic/CrashesPerHour.sql")
_, err := db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())

By("querying the weather data table")
SQL = SQLFromFile("SQL/quickstart/basic/AverageTemp.sql")
_, err = db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())

By("JOINing to see impact of low visibility")
SQL = SQLFromFile("SQL/quickstart/basic/LowVisibility.sql")
_, err = db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())

By("JOINing to see impact of icy weather")
SQL = SQLFromFile("SQL/quickstart/basic/Icy.sql")
_, err = db.Exec(SQL)
Expect(err).ToNot(HaveOccurred())
})
})
})
25 changes: 22 additions & 3 deletions test-harness-docker-compose.yml
Original file line number Diff line number Diff line change
@@ -5,10 +5,29 @@ services:
image: ginkgo
hostname: ginkgo
container_name: ginkgo
environment:
- AWS_S3_ACCESS_KEY=${AWS_S3_ACCESS_KEY}
- AWS_S3_SECRET_KEY=${AWS_S3_SECRET_KEY}
user: root
command: ash
tty: true
build:
context: ci
dockerfile: ginkgo.Dockerfile

minio_mc:
image: minio/mc:latest
entrypoint:
- sh
- -c
- |
until mc ls minio > /dev/null 2>&1; do
sleep 0.5
done
mc alias set myminio http://minio:9000 miniouser miniopassword
mc admin user svcacct add --access-key AAAAAAAAAAAAAAAAAAAA \
--secret-key BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB \
myminio \
miniouser
command: sh
tty: true
depends_on:
- minio

0 comments on commit 4de8c6e

Please sign in to comment.