-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #24 from DanRoscigno/shared_data
Add shared data tests
Showing
12 changed files
with
207 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
name: Test with Shared Data | ||
|
||
on: | ||
schedule: | ||
- cron: "5 9 * * 1" | ||
push: | ||
branches: [ main ] | ||
paths: | ||
- 'ci/**/quickstart/shared_data/*' | ||
- '.github/workflows/test_with_shared_data.yml' | ||
- 'quickstart_shared_data_test.go' | ||
- 'helper.go' | ||
pull_request: | ||
branches: [ main ] | ||
paths: | ||
- 'ci/**/quickstart/shared_data/*' | ||
- '.github/workflows/test_with_shared_data.yml' | ||
- 'quickstart_shared_data_test.go' | ||
- 'helper.go' | ||
|
||
jobs: | ||
build: | ||
|
||
name: Build and test | ||
runs-on: ubuntu-latest | ||
|
||
steps: | ||
# Checkout the repo as this CI needs: | ||
# - the compose file for StarRocks and Ginkgo/Gomega | ||
- name: Checkout Test repo | ||
uses: actions/checkout@v4 | ||
#with: | ||
#path: testing | ||
|
||
# Build the test harness container (Golnag/Ginkgo/Gomega) | ||
- name: Build Ginkgo container | ||
run: | | ||
curl -o shared_data-compose.yml \ | ||
https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/docker-compose.yml | ||
docker compose -f shared_data-compose.yml -f test-harness-docker-compose.yml build | ||
- name: Start StarRocks and MinIO | ||
run: | | ||
docker compose \ | ||
-f shared_data-compose.yml \ | ||
-f test-harness-docker-compose.yml \ | ||
up --detach --wait --wait-timeout 100 | ||
# Any tests that will run against the StarRocks env would be | ||
# launched in steps like this one. Make sure to reset the | ||
# StarRocks environment after each run (remove any tables | ||
# and databases created, and reset any settings to the default) | ||
# | ||
# The ginkgo command uses `--focus-file` to run only the one test | ||
# file. | ||
- name: Test; Shared Data SQL test | ||
if: always() | ||
run: docker compose -f shared_data-compose.yml -f test-harness-docker-compose.yml exec -e SR_FE_HOST='starrocks-fe' test-harness ginkgo -v --focus-file=./quickstart_shared_data_test.go |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,2 @@ | ||
.env | ||
shared_data-compose.yml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
#!/bin/bash | ||
#!/bin/sh | ||
cd /tmp/ | ||
curl --silent --no-buffer \ | ||
-o /tmp/NYPD_Crash_Data.csv \ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
#!/bin/bash | ||
#!/bin/sh | ||
cd /tmp/ | ||
curl --silent --no-buffer \ | ||
-o 72505394728.csv \ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
SHOW STORAGE VOLUMES; | ||
|
||
DESC STORAGE VOLUME builtin_storage_volume; | ||
|
||
CREATE DATABASE IF NOT EXISTS quickstart; | ||
|
||
USE quickstart; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
package docs_test | ||
|
||
import ( | ||
. "github.com/onsi/ginkgo/v2" | ||
. "github.com/onsi/gomega" | ||
) | ||
|
||
var _ = Describe("QuickstartSharedData", func() { | ||
|
||
When("Running the SharedData Quick Start", Ordered, func() { | ||
|
||
// The database is already initialized, and a connection | ||
// is available with the variable `db` which is setup | ||
// in the helpers.go file. | ||
|
||
BeforeAll(func() { | ||
// download the crash data in /tmp/ dir | ||
// https://stackoverflow.com/questions/16703647/why-does-curl-return-error-23-failed-writing-body | ||
By("Downloading the NYPD Crash data") | ||
LongRunningScript("SHELL/quickstart/basic/NYPD_download") | ||
|
||
By("Downloading the NOAA weather data") | ||
LongRunningScript("SHELL/quickstart/basic/Weather_download") | ||
}) | ||
|
||
AfterAll(func() { | ||
By("dropping quickstart DB") | ||
_, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
By("Reset settings") | ||
_, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "3");`) | ||
Expect(err).ToNot(HaveOccurred()) | ||
}) | ||
|
||
It("DDL: Setup quickstart DB", func() { | ||
By("creating a database") | ||
SQL := SQLFromFile("SQL/quickstart/shared_data/shared_data_DDL.sql") | ||
_, err := db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
}) | ||
|
||
// Note: The rest of these tests use the same | ||
// queries as the basic quick start, so they are | ||
// sourced from the basic directory. | ||
|
||
It("DDL: Create quickstart tables", func() { | ||
By("creating the crash data table") | ||
SQL := SQLFromFile("SQL/quickstart/basic/NYPD_table.sql") | ||
_, err := db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
}) | ||
|
||
It("DDL: Create quickstart tables", func() { | ||
|
||
By("creating the weather data table") | ||
SQL := SQLFromFile("SQL/quickstart/basic/Weather_table.sql") | ||
_, err := db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
}) | ||
|
||
It("should be able to load data via stream load", func() { | ||
By("uploading the NYPD crash data") | ||
LongRunningScript("SHELL/quickstart/basic/NYPD_stream_load") | ||
}) | ||
It("should be able to load data via stream load", func() { | ||
|
||
By("uploading the NOAA weather data") | ||
LongRunningScript("SHELL/quickstart/basic/Weather_stream_load") | ||
}) | ||
|
||
It("should be able to query tables", func() { | ||
By("querying the crash data table") | ||
SQL := SQLFromFile("SQL/quickstart/basic/CrashesPerHour.sql") | ||
_, err := db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
By("querying the weather data table") | ||
SQL = SQLFromFile("SQL/quickstart/basic/AverageTemp.sql") | ||
_, err = db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
By("JOINing to see impact of low visibility") | ||
SQL = SQLFromFile("SQL/quickstart/basic/LowVisibility.sql") | ||
_, err = db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
|
||
By("JOINing to see impact of icy weather") | ||
SQL = SQLFromFile("SQL/quickstart/basic/Icy.sql") | ||
_, err = db.Exec(SQL) | ||
Expect(err).ToNot(HaveOccurred()) | ||
}) | ||
}) | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters