Skip to content

Commit

Permalink
update tests
Browse files Browse the repository at this point in the history
Signed-off-by: xxchan <[email protected]>
  • Loading branch information
xxchan committed Apr 22, 2024
1 parent 4924711 commit 57733ab
Show file tree
Hide file tree
Showing 6 changed files with 286 additions and 59 deletions.
40 changes: 30 additions & 10 deletions Makefile.toml
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,7 @@ rm -rf "${PREFIX_PROFILING}"
[tasks.reset-rw]
category = "RiseDev - Start/Stop"
description = "Clean all data in the default database dev of the running RisingWave"
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
dependencies = ["check-and-load-risedev-env-file"]
script = '''
#!/usr/bin/env bash
psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev -c "CREATE DATABASE risedev_tmp;"
Expand Down Expand Up @@ -568,11 +567,16 @@ if [ ! -f "${RC_ENV_FILE}" ]; then
fi
'''

[tasks.check-and-load-risedev-env-file]
private = true
category = "RiseDev - Prepare"
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]

[tasks.psql-env]
category = "RiseDev - Start/Stop"
description = "Dump env configuration for psql"
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
dependencies = ["check-and-load-risedev-env-file"]
script = '''
#!/usr/bin/env bash
cat <<EOF > "${PREFIX_CONFIG}/psql-env"
Expand All @@ -590,8 +594,7 @@ echo " $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)"
[tasks.psql]
category = "RiseDev - Start/Stop"
description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
dependencies = ["check-and-load-risedev-env-file"]
script = '''
#!/usr/bin/env bash
psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev "$@"
Expand All @@ -600,8 +603,7 @@ psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root
[tasks.ctl]
category = "RiseDev - Start/Stop"
description = "Start RiseCtl"
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
dependencies = ["check-and-load-risedev-env-file"]
script = '''
#!/usr/bin/env bash
cargo run -p risingwave_cmd_all --profile "${RISINGWAVE_BUILD_PROFILE}" -- ctl "$@"
Expand Down Expand Up @@ -1313,12 +1315,18 @@ category = "RiseDev - Test - SQLLogicTest"
install_crate = { version = "0.20.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
"--help",
], install_command = "binstall" }
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]
dependencies = ["check-and-load-risedev-env-file"]
command = "sqllogictest"
args = ["${@}"]
description = "🌟 Run SQLLogicTest"

[tasks.slt-clean]
category = "RiseDev - Test - SQLLogicTest"
dependencies = ["clean-kafka", "reset-rw"]
description = "Run SQLLogicTest with a clean environment"
run_task = "slt"
args = ["${@}"]

[tasks.slt-streaming]
category = "RiseDev - Test - SQLLogicTest"
extend = "slt"
Expand Down Expand Up @@ -1433,3 +1441,15 @@ script = """
vars = dump_variables
echo ${vars}
"""

[tasks.show-risedev-env]
description = "Show risedev-env environment variables"
dependencies = ["check-risedev-env-file"]
script = '''
#!/usr/bin/env bash
set -euo pipefail
cat ${PREFIX_CONFIG}/risedev-env
echo "Hint: To load the environment variables into the shell, you may run:"
echo "$(tput setaf 4)\set -a; source ${PREFIX_CONFIG}/risedev-env; set +a$(tput sgr0)"
'''
35 changes: 20 additions & 15 deletions e2e_test/source_inline/commands.toml
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
# This file contains commands used by the tests.

[tasks.source-test-hook]
private = true
dependencies = ["check-risedev-env-file"]
env_files = ["${PREFIX_CONFIG}/risedev-env"]

# Note about the Kafka CLI tooling:
# - Built-in Kafka console tools:
Expand All @@ -16,10 +12,10 @@ env_files = ["${PREFIX_CONFIG}/risedev-env"]
# - rpk:
# Golang based.
# Style example: RPK_BROKERS=localhost:9092 rpk topic create t
[tasks.kafka-hook]
[tasks.check-kafka]
private = true
description = "Check if Kafka is started by RiseDev"
dependencies = ["source-test-hook"]
dependencies = ["check-and-load-risedev-env-file"]
script = '''
#!/usr/bin/env sh
set -e
Expand All @@ -38,13 +34,22 @@ fi
[tasks.clean-kafka]
category = "RiseDev - Test - Source Test - Kafka"
description = "Delete all kafka topics."
dependencies = ["kafka-hook"]
command = "rpk"
args = ["topic", "delete", "-r", "*"]
dependencies = ["check-and-load-risedev-env-file"]
script = '''
#!/usr/bin/env sh
set -e
if [ -n "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" ]; then
echo "Deleting all Kafka topics..."
rpk topic delete -r "*"
else
echo "No Kafka to clean."
fi
'''

[tasks.kafka-topics]
category = "RiseDev - Test - Source Test - Kafka"
dependencies = ["kafka-hook"]
dependencies = ["check-kafka"]
script = """
#!/usr/bin/env sh
set -e
Expand All @@ -53,7 +58,7 @@ ${PREFIX_BIN}/kafka/bin/kafka-topics.sh --bootstrap-server ${RISEDEV_KAFKA_BOOTS

[tasks.kafka-produce]
category = "RiseDev - Test - Source Test - Kafka"
dependencies = ["kafka-hook"]
dependencies = ["check-kafka"]
script = """
#!/usr/bin/env sh
set -e
Expand All @@ -62,7 +67,7 @@ ${PREFIX_BIN}/kafka/bin/kafka-console-producer.sh --bootstrap-server ${RISEDEV_K

[tasks.kafka-consume]
category = "RiseDev - Test - Source Test - Kafka"
dependencies = ["kafka-hook"]
dependencies = ["check-kafka"]
script = """
#!/usr/bin/env sh
set -e
Expand All @@ -71,7 +76,7 @@ ${PREFIX_BIN}/kafka/bin/kafka-console-consumer.sh --bootstrap-server ${RISEDEV_K

[tasks.kafka-consumer-groups]
category = "RiseDev - Test - Source Test - Kafka"
dependencies = ["kafka-hook"]
dependencies = ["check-kafka"]
script = """
#!/usr/bin/env sh
set -e
Expand All @@ -81,7 +86,7 @@ ${PREFIX_BIN}/kafka/bin/kafka-consumer-groups.sh --bootstrap-server ${RISEDEV_KA
# rpk tools
[tasks.rpk]
category = "RiseDev - Test - Source Test - Kafka"
dependencies = ["kafka-hook"]
dependencies = ["check-kafka"]
# check https://docs.redpanda.com/current/reference/rpk/rpk-x-options/ or rpk -X help/list for options
script = """
#!/usr/bin/env sh
Expand All @@ -98,7 +103,7 @@ rpk "$@"
[tasks.redpanda-console]
category = "RiseDev - Test - Source Test - Kafka"
description = "Start Redpanda console (Kafka GUI) at localhost:8080."
dependencies = ["kafka-hook"]
dependencies = ["check-kafka"]
script = '''
#!/usr/bin/env sh
set -e
Expand Down
13 changes: 5 additions & 8 deletions e2e_test/source_inline/kafka/consumer_group.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,8 @@
// zx: A tool for writing better scripts
// https://google.github.io/zx/

const {
mv: mv,
_: _command,
} = minimist(process.argv.slice(3), {
string: ["mv", "topic"],
const { mv: mv, _: _command } = minimist(process.argv.slice(3), {
string: ["mv"],
_: ["list-members", "list-lags"],
});
const command = _command[0];
Expand Down Expand Up @@ -62,7 +59,7 @@ async function list_consumer_group_members(fragment_id) {
const groups = await list_consumer_groups(fragment_id);
return Promise.all(
groups.map(async (group_name) => {
return (await describe_consumer_group(group_name))["MEMBERS"]
return (await describe_consumer_group(group_name))["MEMBERS"];
})
);
}
Expand All @@ -71,14 +68,14 @@ async function list_consumer_group_lags(fragment_id) {
const groups = await list_consumer_groups(fragment_id);
return Promise.all(
groups.map(async (group_name) => {
return (await describe_consumer_group(group_name))["TOTAL-LAG"]
return (await describe_consumer_group(group_name))["TOTAL-LAG"];
})
);
}

const fragment_id = await get_fragment_id_of_mv(mv);
if (command == "list-groups") {
echo`${(await list_consumer_groups(fragment_id))}`;
echo`${await list_consumer_groups(fragment_id)}`;
} else if (command == "list-members") {
echo`${await list_consumer_group_members(fragment_id)}`;
} else if (command == "list-lags") {
Expand Down
65 changes: 65 additions & 0 deletions e2e_test/source_inline/kafka/internal_table.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#!/usr/bin/env zx

// zx: A tool for writing better scripts
// https://google.github.io/zx/

const {
name: job_name,
type: table_type,
_: _command,
} = minimist(process.argv.slice(3), {
string: ["name", "type"],
});

// Return an array of CSV string
async function psql(query) {
return (
await $`
psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev \
--csv --tuples-only -c ${query}
`
)
.toString()
.trim()
.split("\n")
.filter((line) => line.trim() != "");
}

// If `table_type` is null, return all internal tables for the job.
// If `job_name` is null, return all jobs' internal tables.
async function select_internal_table(job_name, table_type) {
// Note: if we have `t1`, and `t1_balabala`, the latter one will also be matched 😄.
const internal_tables = await psql(
`select name from rw_internal_tables where name like '__internal_${job_name}_%_${table_type}_%'`
);
if (internal_tables.length == 0) {
throw new Error(
`No internal tables found for the pattern '__internal_${job_name}_%_${table_type}_%'`
);
}

const res = new Map(
await Promise.all(
internal_tables.map(async (t) => {
let rows = await psql(`select * from ${t}`);
if (rows.length == 0) {
rows = ["(empty)"];
}
return [t, rows];
})
)
);
return res;
}

const tables = await select_internal_table(job_name, table_type);
if (tables.size > 1) {
for (const [table_name, rows] of tables) {
console.log(`Table: ${table_name}`);
console.log(`${rows.join("\n")}`);
}
} else {
for (const [_table_name, rows] of tables) {
console.log(`${rows.join("\n")}`);
}
}
Loading

0 comments on commit 57733ab

Please sign in to comment.