Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into peng/alter-sink-targe…
Browse files Browse the repository at this point in the history
…t-table
  • Loading branch information
shanicky committed Jun 14, 2024
2 parents 8488f35 + 5c24685 commit 5102348
Show file tree
Hide file tree
Showing 182 changed files with 3,434 additions and 1,279 deletions.
20 changes: 18 additions & 2 deletions .github/workflows/stale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,26 @@ jobs:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-issue-message: >
This issue has been open for 60 days with no activity.
Could you please update the status? Feel free to continue discussion or close as not planned.
If you think it is still relevant today, and needs to be done *in the near future*, you can comment to update the status, or just manually remove the `no-issue-activity` label.
You can also confidently close this issue as not planned to keep our backlog clean.
Don't worry if you think the issue is still valuable to continue in the future.
It's searchable and can be reopened when it's time. 😄
stale-pr-message: >
This PR has been open for 60 days with no activity.
Could you please update the status? Feel free to ping a reviewer if you are waiting for review.
If it's blocked by code review, feel free to ping a reviewer or ask someone else to review it.
If you think it is still relevant today, and have time to work on it *in the near future*, you can comment to update the status, or just manually remove the `no-pr-activity` label.
You can also confidently close this PR to keep our backlog clean. (If no further action taken, the PR will be automatically closed after 7 days. Sorry! 🙏)
Don't worry if you think the PR is still valuable to continue in the future.
It's searchable and can be reopened when it's time. 😄
close-pr-message: >
Close this PR as there's no further actions taken after it is marked as stale for 7 days. Sorry! 🙏
You can reopen it when you have time to continue working on it.
stale-issue-label: 'no-issue-activity'
stale-pr-label: 'no-pr-activity'
days-before-close: -1
Expand Down
155 changes: 155 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 11 additions & 1 deletion ci/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: "3.9"
services:
db:
image: postgres:15-alpine
Expand Down Expand Up @@ -100,6 +99,7 @@ services:
- doris-server
- starrocks-fe-server
- starrocks-be-server
- sqlserver-server
volumes:
- ..:/risingwave

Expand Down Expand Up @@ -204,6 +204,16 @@ services:
timeout: 5s
retries: 30

sqlserver-server:
container_name: sqlserver-server
image: mcr.microsoft.com/mssql/server:2022-latest
hostname: sqlserver-server
ports:
- 1433:1433
environment:
ACCEPT_EULA: 'Y'
SA_PASSWORD: 'SomeTestOnly@SA'

starrocks-fe-server:
container_name: starrocks-fe-server
image: starrocks/fe-ubuntu:3.1.7
Expand Down
80 changes: 80 additions & 0 deletions ci/scripts/e2e-sqlserver-sink-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env bash

# Exits as soon as any line fails.
set -euo pipefail

source ci/scripts/common.sh

while getopts 'p:' opt; do
case ${opt} in
p )
profile=$OPTARG
;;
\? )
echo "Invalid Option: -$OPTARG" 1>&2
exit 1
;;
: )
echo "Invalid option: $OPTARG requires an argument" 1>&2
;;
esac
done
shift $((OPTIND -1))

download_and_prepare_rw "$profile" source

echo "--- starting risingwave cluster"
risedev ci-start ci-sink-test
sleep 1

echo "--- create SQL Server table"
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list
apt-get update -y
ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y mssql-tools unixodbc-dev
export PATH="/opt/mssql-tools/bin/:$PATH"
sleep 2

sqlcmd -S sqlserver-server -U SA -P SomeTestOnly@SA -Q "
CREATE DATABASE SinkTest;
GO
USE SinkTest;
CREATE TABLE t_many_data_type (
k1 int, k2 int,
c_boolean bit,
c_int16 smallint,
c_int32 int,
c_int64 bigint,
c_float32 float,
c_float64 float,
c_decimal decimal,
c_date date,
c_time time,
c_timestamp datetime2,
c_timestampz datetime2,
c_nvarchar nvarchar(1024),
c_varbinary varbinary(1024),
PRIMARY KEY (k1,k2));
GO"
sleep 2

echo "--- testing sinks"
sqllogictest -p 4566 -d dev './e2e_test/sink/sqlserver_sink.slt'
sleep 1
sqlcmd -S sqlserver-server -U SA -P SomeTestOnly@SA -h -1 -Q "
SELECT * FROM SinkTest.dbo.t_many_data_type;
GO" > ./query_result.txt

mapfile -t actual < <(tr -s '[:space:]' '\n' < query_result.txt)
actual=("${actual[@]:1}")
expected=(0 0 0 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 1 0 55 55 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF 1 2 0 66 66 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF 1 4 0 2 2 1 1.0 1.0 1 2022-04-08 18:20:49.0000000 2022-03-13 01:00:00.0000000 2022-03-13 01:00:00.0000000 Hello World! 0xDE00BEEF "(4" rows "affected)")

if [[ ${#actual[@]} -eq ${#expected[@]} && ${actual[@]} == ${expected[@]} ]]; then
echo "SQL Server sink check passed"
else
cat ./query_result.txt
echo "The output is not as expected."
fi

echo "--- Kill cluster"
risedev ci-kill
19 changes: 19 additions & 0 deletions ci/workflows/main-cron.yml
Original file line number Diff line number Diff line change
Expand Up @@ -868,6 +868,25 @@ steps:
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end sqlserver sink test"
key: "e2e-sqlserver-sink-tests"
command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-release"
if: |
!(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests"
|| build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/
depends_on:
- "build"
- "build-other"
plugins:
- docker-compose#v5.1.0:
run: sink-test-env
config: ci/docker-compose.yml
mount-buildkite-agent: true
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 10
retry: *auto-retry

- label: "end-to-end pulsar sink test"
key: "e2e-pulsar-sink-tests"
command: "ci/scripts/e2e-pulsar-sink-test.sh -p ci-release"
Expand Down
Loading

0 comments on commit 5102348

Please sign in to comment.