From 396ef03d9387d96a11dc50ee0c6574b3525e4fc2 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Tue, 13 Feb 2024 08:56:38 -0500 Subject: [PATCH] WIP Signed-off-by: DanRoscigno --- .github/workflows/test_with_shared_data.yml | 70 +++++++++++++++ .../shared_data/shared_data_DDL.sql | 7 ++ ci/quickstart_shared_data_test.go | 89 +++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 .github/workflows/test_with_shared_data.yml create mode 100644 ci/SQL/quickstart/shared_data/shared_data_DDL.sql create mode 100644 ci/quickstart_shared_data_test.go diff --git a/.github/workflows/test_with_shared_data.yml b/.github/workflows/test_with_shared_data.yml new file mode 100644 index 0000000..4582f21 --- /dev/null +++ b/.github/workflows/test_with_shared_data.yml @@ -0,0 +1,70 @@ +name: Test with Shared Data + +on: + schedule: + - cron: "5 9 * * 1" + push: + branches: [ main ] + paths: + - 'ci/**/quickstart/shared_data/*' + - '.github/workflows/test_with_shared_data.yml' + - 'quickstart_shared_data_test.go' + - 'helper.go' + pull_request: + branches: [ main ] + paths: + - 'ci/**/quickstart/shared_data/*' + - '.github/workflows/test_with_shared_data.yml' + - 'quickstart_shared_data_test.go' + - 'helper.go' + +jobs: + build: + + name: Build and test + runs-on: ubuntu-latest + + steps: + # Checkout the repo as this CI needs: + # - the compose file for StarRocks and Ginkgo/Gomega + #- name: Checkout Test repo + uses: actions/checkout@v4 + #with: + #path: testing + + #- name: Checkout Demo repo + #uses: actions/checkout@v4 + #with: + #repository: StarRocks/demo + #path: demo + + #- name: Set up Golang + #uses: actions/setup-go@v5 + #with: + #go-version-file: 'testing/ci/go.mod' + + #- name: Install ginkgo + #run: | + #version=$(cat go.mod| grep "ginkgo/v2" | awk '{print $2}') + #go install -v github.com/onsi/ginkgo/v2/ginkgo@$version + #working-directory: ./testing/ci + + - name: Start StarRocks and MinIO + run: | + curl -o shared_data-compose.yml \ + https://raw.githubusercontent.com/StarRocks/demo/master/documentation-samples/quickstart/docker-compose.yml + docker compose up \ + -f shared_data-compose.yml \ + -f test-harness-docker-compose.yml \ + --detach --wait --wait-timeout 60 + #working-directory: demo/documentation-samples/quickstart + + # Any tests that will run against the StarRocks env would be + # launched in steps like this one. Make sure to reset the + # StarRocks environment after each run (remove any tables + # and databases created, and reset any settings to the default) + # + # The ginkgo command uses `--focus-file` to run only the one test + # file. + - name: Test; Shared Data SQL test + run: docker compose -f test-harness-docker-compose.yml exec test-harness ginkgo -v --focus-file=./quickstart_shared_data_test.go diff --git a/ci/SQL/quickstart/shared_data/shared_data_DDL.sql b/ci/SQL/quickstart/shared_data/shared_data_DDL.sql new file mode 100644 index 0000000..24adba3 --- /dev/null +++ b/ci/SQL/quickstart/shared_data/shared_data_DDL.sql @@ -0,0 +1,7 @@ +SHOW STORAGE VOLUMES; + +DESC STORAGE VOLUME builtin_storage_volume; + +CREATE DATABASE IF NOT EXISTS quickstart; + +USE quickstart; diff --git a/ci/quickstart_shared_data_test.go b/ci/quickstart_shared_data_test.go new file mode 100644 index 0000000..7d6bd9f --- /dev/null +++ b/ci/quickstart_shared_data_test.go @@ -0,0 +1,89 @@ +package docs_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("QuickstartSharedData", func() { + + When("Running the SharedData Quick Start", Ordered, func() { + + // The database is already initialized, and a connection + // is available with the variable `db` which is setup + // in the helpers.go file. + + BeforeAll(func() { + // download the crash data in /tmp/ dir + // https://stackoverflow.com/questions/16703647/why-does-curl-return-error-23-failed-writing-body + By("Downloading the NYPD Crash data") + LongRunningScript("SHELL/quickstart/basic/NYPD_download") + + By("Downloading the NOAA weather data") + LongRunningScript("SHELL/quickstart/basic/Weather_download") + }) + + AfterAll(func() { + By("dropping quickstart DB") + _, err := db.Exec(`DROP DATABASE IF EXISTS quickstart`) + Expect(err).ToNot(HaveOccurred()) + + By("Reset settings") + _, err = db.Exec(`ADMIN SET FRONTEND CONFIG ('default_replication_num' = "3");`) + Expect(err).ToNot(HaveOccurred()) + }) + + It("DDL: Setup quickstart DB", func() { + By("creating a database") + SQL := SQLFromFile("SQL/quickstart/shared_data/shared_data_DDL.sql") + _, err := db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + }) + + // Note: The rest of these tests use the same + // queries as the basic quick start, so they are + // sourced from the basic directory. + + It("DDL: Create quickstart tables", func() { + By("creating the crash data table") + SQL := SQLFromFile("SQL/quickstart/basic/NYPD_table.sql") + _, err := db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("creating the weather data table") + SQL = SQLFromFile("SQL/quickstart/basic/Weather_table.sql") + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should be able to load data via stream load", func() { + By("uploading the NYPD crash data") + LongRunningScript("SHELL/quickstart/basic/NYPD_stream_load") + + By("uploading the NOAA weather data") + LongRunningScript("SHELL/quickstart/basic/Weather_stream_load") + }) + + It("should be able to query tables", func() { + By("querying the crash data table") + SQL := SQLFromFile("SQL/quickstart/basic/CrashesPerHour.sql") + _, err := db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("querying the weather data table") + SQL = SQLFromFile("SQL/quickstart/basic/AverageTemp.sql") + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("JOINing to see impact of low visibility") + SQL = SQLFromFile("SQL/quickstart/basic/LowVisibility.sql") + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + + By("JOINing to see impact of icy weather") + SQL = SQLFromFile("SQL/quickstart/basic/Icy.sql") + _, err = db.Exec(SQL) + Expect(err).ToNot(HaveOccurred()) + }) + }) +})