From f217edde107fbf6369d1bd1088b7e35e6c6ba912 Mon Sep 17 00:00:00 2001 From: "Zhanxiang (Patrick) Huang" Date: Mon, 14 Oct 2024 21:06:52 +0800 Subject: [PATCH 1/7] feat: improve observability for subscription/cursor SHOW commands (#18896) --- src/frontend/src/handler/show.rs | 122 ++++++++++++++++++--- src/frontend/src/session/cursor_manager.rs | 99 +++++++++-------- 2 files changed, 158 insertions(+), 63 deletions(-) diff --git a/src/frontend/src/handler/show.rs b/src/frontend/src/handler/show.rs index 1821ccc289ebc..0da3e76ce3fe6 100644 --- a/src/frontend/src/handler/show.rs +++ b/src/frontend/src/handler/show.rs @@ -31,11 +31,12 @@ use risingwave_sqlparser::ast::{ display_comma_separated, Ident, ObjectName, ShowCreateType, ShowObject, ShowStatementFilter, }; -use super::{fields_to_descriptors, PgResponseStream, RwPgResponse, RwPgResponseBuilderExt}; +use super::{fields_to_descriptors, RwPgResponse, RwPgResponseBuilderExt}; use crate::binder::{Binder, Relation}; use crate::catalog::{CatalogError, IndexCatalog}; use crate::error::Result; use crate::handler::HandlerArgs; +use crate::session::cursor_manager::SubscriptionCursor; use crate::session::SessionImpl; pub fn get_columns_from_table( @@ -247,6 +248,36 @@ struct ShowCreateObjectRow { create_sql: String, } +#[derive(Fields)] +#[fields(style = "Title Case")] +struct ShowSubscriptionRow { + name: String, + retention_seconds: i64, +} + +#[derive(Fields)] +#[fields(style = "Title Case")] +struct ShowCursorRow { + session_id: String, + user: String, + host: String, + database: String, + cursor_name: String, +} + +#[derive(Fields)] +#[fields(style = "Title Case")] +struct ShowSubscriptionCursorRow { + session_id: String, + user: String, + host: String, + database: String, + cursor_name: String, + subscription_name: String, + state: String, + idle_duration_ms: i64, +} + /// Infer the row description for different show objects. pub fn infer_show_object(objects: &ShowObject) -> Vec { fields_to_descriptors(match objects { @@ -327,12 +358,20 @@ pub async fn handle_show_object( .iter_sink() .map(|t| t.name.clone()) .collect(), - ShowObject::Subscription { schema } => catalog_reader - .read_guard() - .get_schema_by_name(session.database(), &schema_or_default(&schema))? - .iter_subscription() - .map(|t| t.name.clone()) - .collect(), + ShowObject::Subscription { schema } => { + let rows = catalog_reader + .read_guard() + .get_schema_by_name(session.database(), &schema_or_default(&schema))? + .iter_subscription() + .map(|t| ShowSubscriptionRow { + name: t.name.clone(), + retention_seconds: t.retention_seconds as i64, + }) + .collect_vec(); + return Ok(PgResponse::builder(StatementType::SHOW_COMMAND) + .rows(rows) + .into()); + } ShowObject::Secret { schema } => catalog_reader .read_guard() .get_schema_by_name(session.database(), &schema_or_default(&schema))? @@ -483,20 +522,71 @@ pub async fn handle_show_object( .into()); } ShowObject::Cursor => { - let (rows, pg_descs) = session.get_cursor_manager().get_all_query_cursors().await; + let sessions = session + .env() + .sessions_map() + .read() + .values() + .cloned() + .collect_vec(); + let mut rows = vec![]; + for s in sessions { + let session_id = format!("{}", s.id().0); + let user = s.user_name().to_owned(); + let host = format!("{}", s.peer_addr()); + let database = s.database().to_owned(); + + s.get_cursor_manager() + .iter_query_cursors(|cursor_name: &String, _| { + rows.push(ShowCursorRow { + session_id: session_id.clone(), + user: user.clone(), + host: host.clone(), + database: database.clone(), + cursor_name: cursor_name.to_owned(), + }); + }) + .await; + } return Ok(PgResponse::builder(StatementType::SHOW_COMMAND) - .row_cnt_opt(Some(rows.len() as i32)) - .values(PgResponseStream::from(rows), pg_descs) + .rows(rows) .into()); } ShowObject::SubscriptionCursor => { - let (rows, pg_descs) = session - .get_cursor_manager() - .get_all_subscription_cursors() - .await; + let sessions = session + .env() + .sessions_map() + .read() + .values() + .cloned() + .collect_vec(); + let mut rows = vec![]; + for s in sessions { + let ssession_id = format!("{}", s.id().0); + let user = s.user_name().to_owned(); + let host = format!("{}", s.peer_addr()); + let database = s.database().to_owned(); + + s.get_cursor_manager() + .iter_subscription_cursors( + |cursor_name: &String, cursor: &SubscriptionCursor| { + rows.push(ShowSubscriptionCursorRow { + session_id: ssession_id.clone(), + user: user.clone(), + host: host.clone(), + database: database.clone(), + cursor_name: cursor_name.to_owned(), + subscription_name: cursor.subscription_name().to_owned(), + state: cursor.state_info_string(), + idle_duration_ms: cursor.idle_duration().as_millis() as i64, + }); + }, + ) + .await; + } + return Ok(PgResponse::builder(StatementType::SHOW_COMMAND) - .row_cnt_opt(Some(rows.len() as i32)) - .values(PgResponseStream::from(rows), pg_descs) + .rows(rows) .into()); } }; diff --git a/src/frontend/src/session/cursor_manager.rs b/src/frontend/src/session/cursor_manager.rs index 7cfdc0aa012a0..4115da5f3025e 100644 --- a/src/frontend/src/session/cursor_manager.rs +++ b/src/frontend/src/session/cursor_manager.rs @@ -15,6 +15,7 @@ use core::mem; use core::time::Duration; use std::collections::{HashMap, HashSet, VecDeque}; +use std::fmt::{Display, Formatter}; use std::rc::Rc; use std::sync::Arc; use std::time::Instant; @@ -225,6 +226,34 @@ enum State { Invalid, } +impl Display for State { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + State::InitLogStoreQuery { + seek_timestamp, + expected_timestamp, + } => write!( + f, + "InitLogStoreQuery {{ seek_timestamp: {}, expected_timestamp: {:?} }}", + seek_timestamp, expected_timestamp + ), + State::Fetch { + from_snapshot, + rw_timestamp, + expected_timestamp, + remaining_rows, + init_query_timer, + .. + } => write!( + f, + "Fetch {{ from_snapshot: {}, rw_timestamp: {}, expected_timestamp: {:?}, cached rows: {}, query init at {}ms before }}", + from_snapshot, rw_timestamp, expected_timestamp, remaining_rows.len(), init_query_timer.elapsed().as_millis() + ), + State::Invalid => write!(f, "Invalid"), + } + } +} + pub struct SubscriptionCursor { cursor_name: String, subscription: Arc, @@ -727,6 +756,18 @@ impl SubscriptionCursor { } chunk_stream.init_row_stream(&fields, &formats, session); } + + pub fn idle_duration(&self) -> Duration { + self.last_fetch.elapsed() + } + + pub fn subscription_name(&self) -> &str { + self.subscription.name.as_str() + } + + pub fn state_info_string(&self) -> String { + format!("{}", self.state) + } } pub struct CursorManager { @@ -876,63 +917,27 @@ impl CursorManager { } } - pub async fn get_all_query_cursors(&self) -> (Vec, Vec) { - let cursor_names = self - .cursor_map + pub async fn iter_query_cursors(&self, mut f: impl FnMut(&String, &QueryCursor)) { + self.cursor_map .lock() .await .iter() - .filter_map(|(currsor_name, cursor)| { - if let Cursor::Query(_cursor) = cursor { - let cursor_name = vec![Some(Bytes::from(currsor_name.clone().into_bytes()))]; - Some(Row::new(cursor_name)) - } else { - None + .for_each(|(cursor_name, cursor)| { + if let Cursor::Query(cursor) = cursor { + f(cursor_name, cursor) } - }) - .collect(); - ( - cursor_names, - vec![PgFieldDescriptor::new( - "Name".to_string(), - DataType::Varchar.to_oid(), - DataType::Varchar.type_len(), - )], - ) + }); } - pub async fn get_all_subscription_cursors(&self) -> (Vec, Vec) { - let cursors = self - .cursor_map + pub async fn iter_subscription_cursors(&self, mut f: impl FnMut(&String, &SubscriptionCursor)) { + self.cursor_map .lock() .await .iter() - .filter_map(|(cursor_name, cursor)| { + .for_each(|(cursor_name, cursor)| { if let Cursor::Subscription(cursor) = cursor { - let cursors = vec![ - Some(Bytes::from(cursor_name.clone().into_bytes())), - Some(Bytes::from(cursor.subscription.name.clone().into_bytes())), - ]; - Some(Row::new(cursors)) - } else { - None + f(cursor_name, cursor) } - }) - .collect(); - ( - cursors, - vec![ - PgFieldDescriptor::new( - "Name".to_string(), - DataType::Varchar.to_oid(), - DataType::Varchar.type_len(), - ), - PgFieldDescriptor::new( - "SubscriptionName".to_string(), - DataType::Varchar.to_oid(), - DataType::Varchar.type_len(), - ), - ], - ) + }); } } From 883bcb1e781e8909c3c9227584c3611a2232ad39 Mon Sep 17 00:00:00 2001 From: xxchan Date: Mon, 14 Oct 2024 21:14:33 +0800 Subject: [PATCH 2/7] refactor(test): mv `scripts/source` `e2e_test/source_legacy/basic/scripts` (#18891) --- .gitattributes | 2 +- ci/scripts/deterministic-e2e-test.sh | 4 ++-- ci/scripts/deterministic-recovery-test.sh | 2 +- ci/scripts/e2e-source-test.sh | 4 ++-- .../source_inline/kafka/avro/name_strategy.slt | 6 +++--- e2e_test/source_legacy/README.md | 4 ++-- e2e_test/source_legacy/basic/scripts/README.md | 12 ++++++++++++ .../basic/scripts}/prepare_ci_kafka.sh | 15 +++++++-------- .../basic/scripts}/schema_registry_producer.py | 0 .../source_legacy/basic/scripts}/test_data.zip | Bin .../scripts}/test_data/avro_complex_schema_bin.1 | Bin .../scripts}/test_data/avro_simple_schema_bin.1 | Bin .../basic/scripts}/test_data/bug_bash.1 | 0 .../scripts}/test_data/canal_json_double_field.1 | 0 .../basic/scripts}/test_data/cannal_json.1 | 0 .../test_data/debezium_compact_avro_json.1 | 0 .../test_data/debezium_ignore_case_json.1 | 0 .../basic/scripts}/test_data/debezium_log.1 | 0 .../test_data/debezium_log_no_schema_field.1 | 0 .../basic/scripts}/test_data/debezium_mess_key.1 | 0 .../test_data/debezium_mongo_json_customers.1 | 0 ...bezium_mongo_json_customers_no_schema_field.1 | 0 .../test_data/debezium_non_compact_avro_json.1 | 0 .../basic/scripts}/test_data/json_bytea.1 | 0 .../basic/scripts}/test_data/json_c.1 | 0 .../test_data/json_schema_without_schema.1 | 0 .../test_data/json_timestamptz_handling_mode.1 | 0 .../basic/scripts}/test_data/kafka_1_csv_topic.1 | 0 .../test_data/kafka_1_partition_mv_topic.1 | 0 .../scripts}/test_data/kafka_1_partition_topic.1 | 0 .../scripts}/test_data/kafka_2_partition_topic.2 | 0 .../scripts}/test_data/kafka_3_partition_topic.3 | 0 .../scripts}/test_data/kafka_4_partition_topic.4 | 0 .../kafka_4_partition_topic_generated_columns.1 | 0 .../kafka_4_partition_topic_with_100_message.4 | 0 .../basic/scripts}/test_data/kafka_json_schema.1 | 0 .../test_data/kafka_source_format_bytes.1 | 0 .../test_data/kafka_upsert_json_schema.1 | 0 .../basic/scripts}/test_data/maxwell_json.1 | 0 .../basic/scripts}/test_data/proto_c_bin.1 | 0 .../basic/scripts}/test_data/sink_debezium.1 | 0 .../basic/scripts}/test_data/sink_target.1 | 0 .../basic/scripts}/test_data/upsert_json.1 | 0 .../test_data/upsert_student_avro_json.1 | 0 ...t_student_key_not_subset_of_value_avro_json.1 | 0 .../basic/scripts}/test_data/weiling.1 | 0 scripts/source/README.md | 2 +- .../codec/tests/integration_tests/avro.rs | 2 +- 48 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 e2e_test/source_legacy/basic/scripts/README.md rename {scripts/source => e2e_test/source_legacy/basic/scripts}/prepare_ci_kafka.sh (76%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/schema_registry_producer.py (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data.zip (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/avro_complex_schema_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/avro_simple_schema_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/bug_bash.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/canal_json_double_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/cannal_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_compact_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_ignore_case_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_log.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_log_no_schema_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_mess_key.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_mongo_json_customers.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_mongo_json_customers_no_schema_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_non_compact_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_bytea.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_c.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_schema_without_schema.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_timestamptz_handling_mode.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_1_csv_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_1_partition_mv_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_1_partition_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_2_partition_topic.2 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_3_partition_topic.3 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_4_partition_topic.4 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_4_partition_topic_generated_columns.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_4_partition_topic_with_100_message.4 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_json_schema.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_source_format_bytes.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_upsert_json_schema.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/maxwell_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/proto_c_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/sink_debezium.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/sink_target.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/upsert_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/upsert_student_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/upsert_student_key_not_subset_of_value_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/weiling.1 (100%) diff --git a/.gitattributes b/.gitattributes index becfd0ec71edf..64e8a463c51f6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,6 @@ # regression test data src/tests/regress/data/** linguist-vendored # source test data -scripts/source/test_data/** linguist-vendored +e2e_test/source_legacy/basic/scripts/test_data/** linguist-vendored # generated grafana dashboard grafana/risingwave-dashboard.json linguist-generated diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh index e7c7b21310939..b0965688003c2 100755 --- a/ci/scripts/deterministic-e2e-test.sh +++ b/ci/scripts/deterministic-e2e-test.sh @@ -11,7 +11,7 @@ download-and-decompress-artifact risingwave_simulation . chmod +x ./risingwave_simulation echo "--- Extract data for Kafka" -pushd ./scripts/source/ +pushd ./e2e_test/source_legacy/basic/scripts/ mkdir -p ./test_data unzip -o test_data.zip -d . popd @@ -39,7 +39,7 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, batch" seq "$TEST_NUM" | parallel './risingwave_simulation ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, kafka source" -seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./scripts/source/test_data ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log' +seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, streaming" seq "$TEST_NUM" | parallel './risingwave_simulation -j 16 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/parallel-streaming-{}.log && rm $LOGDIR/parallel-streaming-{}.log' diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 62d2250af96ec..bf1f28861c24a 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -93,6 +93,6 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, kafka source seq "$TEST_NUM" | parallel './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ ---kafka-datadir=./scripts/source/test_data \ +--kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data \ ${EXTRA_ARGS:-} \ ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 5c53f84c24858..9f261f7b4cf08 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -148,11 +148,11 @@ echo "--- Kill cluster" risedev ci-kill export RISINGWAVE_CI=true -echo "--- e2e, ci-kafka-plus-pubsub, kafka and pubsub source" +echo "--- e2e, ci-kafka-plus-pubsub, legacy kafka tests" export RUST_MIN_STACK=4194304 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ risedev ci-start ci-kafka -./scripts/source/prepare_ci_kafka.sh +./e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh risedev slt './e2e_test/source_legacy/basic/*.slt' risedev slt './e2e_test/source_legacy/basic/old_row_format_syntax/*.slt' diff --git a/e2e_test/source_inline/kafka/avro/name_strategy.slt b/e2e_test/source_inline/kafka/avro/name_strategy.slt index 212c79e6f31a8..f0b49a36ea284 100644 --- a/e2e_test/source_inline/kafka/avro/name_strategy.slt +++ b/e2e_test/source_inline/kafka/avro/name_strategy.slt @@ -25,7 +25,7 @@ create source s1 () with ( # Currently we are abusing this test case to also test data types. system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro" +python3 e2e_test/source_legacy/basic/scripts/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro" statement ok CREATE TABLE t_topic ( primary key (rw_key) ) @@ -44,7 +44,7 @@ FORMAT UPSERT ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}'); ## topic: upsert_avro_json-record, key subject: string, value subject: CPLM.OBJ_ATTRIBUTE_VALUE system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro" +python3 e2e_test/source_legacy/basic/scripts/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro" statement error key\.message @@ -80,7 +80,7 @@ create table t_record_format_plain () with ( ## key subject: upsert_avro_json-topic-record-string ## value subject: upsert_avro_json-topic-record-CPLM.OBJ_ATTRIBUTE_VALUE system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro" +python3 e2e_test/source_legacy/basic/scripts/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro" diff --git a/e2e_test/source_legacy/README.md b/e2e_test/source_legacy/README.md index 611619cb13423..0b2d817df988c 100644 --- a/e2e_test/source_legacy/README.md +++ b/e2e_test/source_legacy/README.md @@ -7,8 +7,8 @@ Test in this directory needs some prior setup. -See also `ci/scripts/e2e-source-test.sh`, and `scripts/source` +See also `ci/scripts/e2e-source-test.sh`, and `e2e_test/source_legacy/basic/scripts` ## Kafka -`scripts/source/test_data` contains the data. Filename's convention is `.`. +`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `.`. diff --git a/e2e_test/source_legacy/basic/scripts/README.md b/e2e_test/source_legacy/basic/scripts/README.md new file mode 100644 index 0000000000000..928394a43007e --- /dev/null +++ b/e2e_test/source_legacy/basic/scripts/README.md @@ -0,0 +1,12 @@ +This folder contains scripts to prepare data for testing sources. + +## Kafka + +`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `.`. + +- If `` ends with `bin`, the whole file is a message with binary data. +- If `` ends with `avro_json` or `json_schema`: + - The first line is the schema. Key and value are separated by `^`. + - The rest of the lines are messages in JSON format. Key and value are separated by `^`. + - Produced to Kafka with `schema_registry_producer.py` (serialized to Avro or JSON) +- Otherwise, each line is a message, and key/value is separated by `^`. diff --git a/scripts/source/prepare_ci_kafka.sh b/e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh similarity index 76% rename from scripts/source/prepare_ci_kafka.sh rename to e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh index 27c88e6e63af2..75abd7095a925 100755 --- a/scripts/source/prepare_ci_kafka.sh +++ b/e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh @@ -2,18 +2,19 @@ # Exits as soon as any line fails. set -e +export CARGO_MAKE_PRINT_TIME_SUMMARY=false SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" -cd "$SCRIPT_PATH/.." || exit 1 -# cwd is /scripts +# SCRIPT_PATH is e2e_test/source_legacy/kafka/script/ +# cwd is e2e_test/source_legacy/kafka/ echo "$SCRIPT_PATH" +cd "$SCRIPT_PATH" -source ../.risingwave/config/risedev-env +source ../../../../.risingwave/config/risedev-env if [ "$1" == "compress" ]; then echo "Compress test_data/ into test_data.zip" - cd ./source zip_file=test_data.zip if [ -f "$zip_file" ]; then rm "$zip_file" @@ -23,10 +24,8 @@ if [ "$1" == "compress" ]; then fi echo "--- Extract data for Kafka" -cd ./source/ mkdir -p ./test_data/ch_benchmark/ unzip -o test_data.zip -d . -cd .. echo "path:${SCRIPT_PATH}/test_data/**/*" @@ -58,9 +57,9 @@ for filename in $kafka_data_files; do if [[ "$topic" = *bin ]]; then kcat -P -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t "$topic" "$filename" elif [[ "$topic" = *avro_json ]]; then - python3 source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "avro" + python3 schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "avro" elif [[ "$topic" = *json_schema ]]; then - python3 source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "json" + python3 schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "json" else cat "$filename" | kcat -P -K ^ -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t "$topic" fi diff --git a/scripts/source/schema_registry_producer.py b/e2e_test/source_legacy/basic/scripts/schema_registry_producer.py similarity index 100% rename from scripts/source/schema_registry_producer.py rename to e2e_test/source_legacy/basic/scripts/schema_registry_producer.py diff --git a/scripts/source/test_data.zip b/e2e_test/source_legacy/basic/scripts/test_data.zip similarity index 100% rename from scripts/source/test_data.zip rename to e2e_test/source_legacy/basic/scripts/test_data.zip diff --git a/scripts/source/test_data/avro_complex_schema_bin.1 b/e2e_test/source_legacy/basic/scripts/test_data/avro_complex_schema_bin.1 similarity index 100% rename from scripts/source/test_data/avro_complex_schema_bin.1 rename to e2e_test/source_legacy/basic/scripts/test_data/avro_complex_schema_bin.1 diff --git a/scripts/source/test_data/avro_simple_schema_bin.1 b/e2e_test/source_legacy/basic/scripts/test_data/avro_simple_schema_bin.1 similarity index 100% rename from scripts/source/test_data/avro_simple_schema_bin.1 rename to e2e_test/source_legacy/basic/scripts/test_data/avro_simple_schema_bin.1 diff --git a/scripts/source/test_data/bug_bash.1 b/e2e_test/source_legacy/basic/scripts/test_data/bug_bash.1 similarity index 100% rename from scripts/source/test_data/bug_bash.1 rename to e2e_test/source_legacy/basic/scripts/test_data/bug_bash.1 diff --git a/scripts/source/test_data/canal_json_double_field.1 b/e2e_test/source_legacy/basic/scripts/test_data/canal_json_double_field.1 similarity index 100% rename from scripts/source/test_data/canal_json_double_field.1 rename to e2e_test/source_legacy/basic/scripts/test_data/canal_json_double_field.1 diff --git a/scripts/source/test_data/cannal_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/cannal_json.1 similarity index 100% rename from scripts/source/test_data/cannal_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/cannal_json.1 diff --git a/scripts/source/test_data/debezium_compact_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_compact_avro_json.1 similarity index 100% rename from scripts/source/test_data/debezium_compact_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_compact_avro_json.1 diff --git a/scripts/source/test_data/debezium_ignore_case_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_ignore_case_json.1 similarity index 100% rename from scripts/source/test_data/debezium_ignore_case_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_ignore_case_json.1 diff --git a/scripts/source/test_data/debezium_log.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_log.1 similarity index 100% rename from scripts/source/test_data/debezium_log.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_log.1 diff --git a/scripts/source/test_data/debezium_log_no_schema_field.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_log_no_schema_field.1 similarity index 100% rename from scripts/source/test_data/debezium_log_no_schema_field.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_log_no_schema_field.1 diff --git a/scripts/source/test_data/debezium_mess_key.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_mess_key.1 similarity index 100% rename from scripts/source/test_data/debezium_mess_key.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_mess_key.1 diff --git a/scripts/source/test_data/debezium_mongo_json_customers.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers.1 similarity index 100% rename from scripts/source/test_data/debezium_mongo_json_customers.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers.1 diff --git a/scripts/source/test_data/debezium_mongo_json_customers_no_schema_field.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers_no_schema_field.1 similarity index 100% rename from scripts/source/test_data/debezium_mongo_json_customers_no_schema_field.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers_no_schema_field.1 diff --git a/scripts/source/test_data/debezium_non_compact_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_non_compact_avro_json.1 similarity index 100% rename from scripts/source/test_data/debezium_non_compact_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_non_compact_avro_json.1 diff --git a/scripts/source/test_data/json_bytea.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_bytea.1 similarity index 100% rename from scripts/source/test_data/json_bytea.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_bytea.1 diff --git a/scripts/source/test_data/json_c.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_c.1 similarity index 100% rename from scripts/source/test_data/json_c.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_c.1 diff --git a/scripts/source/test_data/json_schema_without_schema.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_schema_without_schema.1 similarity index 100% rename from scripts/source/test_data/json_schema_without_schema.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_schema_without_schema.1 diff --git a/scripts/source/test_data/json_timestamptz_handling_mode.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_timestamptz_handling_mode.1 similarity index 100% rename from scripts/source/test_data/json_timestamptz_handling_mode.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_timestamptz_handling_mode.1 diff --git a/scripts/source/test_data/kafka_1_csv_topic.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_1_csv_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_csv_topic.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_1_csv_topic.1 diff --git a/scripts/source/test_data/kafka_1_partition_mv_topic.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_mv_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_partition_mv_topic.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_mv_topic.1 diff --git a/scripts/source/test_data/kafka_1_partition_topic.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_partition_topic.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_topic.1 diff --git a/scripts/source/test_data/kafka_2_partition_topic.2 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_2_partition_topic.2 similarity index 100% rename from scripts/source/test_data/kafka_2_partition_topic.2 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_2_partition_topic.2 diff --git a/scripts/source/test_data/kafka_3_partition_topic.3 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_3_partition_topic.3 similarity index 100% rename from scripts/source/test_data/kafka_3_partition_topic.3 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_3_partition_topic.3 diff --git a/scripts/source/test_data/kafka_4_partition_topic.4 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic.4 similarity index 100% rename from scripts/source/test_data/kafka_4_partition_topic.4 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic.4 diff --git a/scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_generated_columns.1 similarity index 100% rename from scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_generated_columns.1 diff --git a/scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_with_100_message.4 similarity index 100% rename from scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_with_100_message.4 diff --git a/scripts/source/test_data/kafka_json_schema.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_json_schema.1 similarity index 100% rename from scripts/source/test_data/kafka_json_schema.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_json_schema.1 diff --git a/scripts/source/test_data/kafka_source_format_bytes.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_source_format_bytes.1 similarity index 100% rename from scripts/source/test_data/kafka_source_format_bytes.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_source_format_bytes.1 diff --git a/scripts/source/test_data/kafka_upsert_json_schema.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_upsert_json_schema.1 similarity index 100% rename from scripts/source/test_data/kafka_upsert_json_schema.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_upsert_json_schema.1 diff --git a/scripts/source/test_data/maxwell_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/maxwell_json.1 similarity index 100% rename from scripts/source/test_data/maxwell_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/maxwell_json.1 diff --git a/scripts/source/test_data/proto_c_bin.1 b/e2e_test/source_legacy/basic/scripts/test_data/proto_c_bin.1 similarity index 100% rename from scripts/source/test_data/proto_c_bin.1 rename to e2e_test/source_legacy/basic/scripts/test_data/proto_c_bin.1 diff --git a/scripts/source/test_data/sink_debezium.1 b/e2e_test/source_legacy/basic/scripts/test_data/sink_debezium.1 similarity index 100% rename from scripts/source/test_data/sink_debezium.1 rename to e2e_test/source_legacy/basic/scripts/test_data/sink_debezium.1 diff --git a/scripts/source/test_data/sink_target.1 b/e2e_test/source_legacy/basic/scripts/test_data/sink_target.1 similarity index 100% rename from scripts/source/test_data/sink_target.1 rename to e2e_test/source_legacy/basic/scripts/test_data/sink_target.1 diff --git a/scripts/source/test_data/upsert_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/upsert_json.1 similarity index 100% rename from scripts/source/test_data/upsert_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/upsert_json.1 diff --git a/scripts/source/test_data/upsert_student_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/upsert_student_avro_json.1 similarity index 100% rename from scripts/source/test_data/upsert_student_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/upsert_student_avro_json.1 diff --git a/scripts/source/test_data/upsert_student_key_not_subset_of_value_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/upsert_student_key_not_subset_of_value_avro_json.1 similarity index 100% rename from scripts/source/test_data/upsert_student_key_not_subset_of_value_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/upsert_student_key_not_subset_of_value_avro_json.1 diff --git a/scripts/source/test_data/weiling.1 b/e2e_test/source_legacy/basic/scripts/test_data/weiling.1 similarity index 100% rename from scripts/source/test_data/weiling.1 rename to e2e_test/source_legacy/basic/scripts/test_data/weiling.1 diff --git a/scripts/source/README.md b/scripts/source/README.md index 311c058a8230b..928394a43007e 100644 --- a/scripts/source/README.md +++ b/scripts/source/README.md @@ -2,7 +2,7 @@ This folder contains scripts to prepare data for testing sources. ## Kafka -`scripts/source/test_data` contains the data. Filename's convention is `.`. +`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `.`. - If `` ends with `bin`, the whole file is a message with binary data. - If `` ends with `avro_json` or `json_schema`: diff --git a/src/connector/codec/tests/integration_tests/avro.rs b/src/connector/codec/tests/integration_tests/avro.rs index ab1df6e7e82b8..d916dc1aba426 100644 --- a/src/connector/codec/tests/integration_tests/avro.rs +++ b/src/connector/codec/tests/integration_tests/avro.rs @@ -119,7 +119,7 @@ fn check( expected_risingwave_data.assert_eq(&format!("{}", data_str.iter().format("\n----\n"))); } -// This corresponds to legacy `scripts/source/test_data/avro_simple_schema_bin.1`. TODO: remove that file. +// This corresponds to legacy `e2e_test/source_legacy/basic/scripts/test_data/avro_simple_schema_bin.1`. TODO: remove that file. #[test] fn test_simple() { check( From 0045fd6f2d3de1338daaf223960e38d8da924ea6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:30:33 +0800 Subject: [PATCH 3/7] chore(deps): bump typed-builder from 0.18.2 to 0.20.0 (#18850) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- src/connector/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a95ad5015815..2c6c028c3deb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11047,7 +11047,7 @@ dependencies = [ "tracing", "tracing-subscriber", "tracing-test", - "typed-builder 0.18.2", + "typed-builder 0.20.0", "url", "urlencoding", "uuid", diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index e89f1d123485f..984ddf18ecf09 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -174,7 +174,7 @@ tokio-stream = { workspace = true } tokio-util = { workspace = true, features = ["codec", "io"] } tonic = { workspace = true } tracing = "0.1" -typed-builder = "^0.18" +typed-builder = "^0.20" url = "2" urlencoding = "2" uuid = { version = "1", features = ["v4", "fast-rng"] } From 67514c1cb2944b298a39608bcf3daeca4318bea9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:36:29 +0000 Subject: [PATCH 4/7] chore(deps): Bump pbjson-build from 0.5.1 to 0.7.0 (#18781) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ src/prost/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c6c028c3deb3..7ca90675afc0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8829,14 +8829,14 @@ dependencies = [ [[package]] name = "pbjson-build" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbb7b706f2afc610f3853550cdbbf6372fd324824a087806bd4480ea4996e24" +checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", - "prost 0.11.9", - "prost-types 0.11.9", + "heck 0.5.0", + "itertools 0.13.0", + "prost 0.13.1", + "prost-types 0.13.1", ] [[package]] diff --git a/src/prost/Cargo.toml b/src/prost/Cargo.toml index 6fc669a824224..33bd04640dd4b 100644 --- a/src/prost/Cargo.toml +++ b/src/prost/Cargo.toml @@ -23,7 +23,7 @@ workspace-hack = { path = "../workspace-hack" } [build-dependencies] fs-err = "2.11" -pbjson-build = "0.5" +pbjson-build = "0.7" prost-build = { workspace = true } tonic-build = { workspace = true } walkdir = "2" From a6f2b88b1bd81e7216fdb85035cc2ad9d0288c95 Mon Sep 17 00:00:00 2001 From: lmatz Date: Mon, 14 Oct 2024 21:46:51 +0800 Subject: [PATCH 5/7] fix: misused source row count metric (#18902) --- src/connector/src/source/common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/src/source/common.rs b/src/connector/src/source/common.rs index 80aacff2899c7..1ad9d4d639935 100644 --- a/src/connector/src/source/common.rs +++ b/src/connector/src/source/common.rs @@ -82,7 +82,7 @@ pub(crate) async fn into_chunk_stream( .flat_map(|msg| msg.payload.as_ref().map(|p| p.len() as u64)) .sum(); - partition_input_count + partition_bytes_count .get_mut(&split_id) .unwrap() .inc_by(sum_bytes); From adc48b579905be0fefafb0064cdf3a50ffe10631 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:04:33 +0000 Subject: [PATCH 6/7] chore(deps): Bump commons-io:commons-io from 2.11.0 to 2.14.0 in /java (#18791) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: xxchan --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 94de4528d24ac..fd01628204b29 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -71,7 +71,7 @@ 2.20.0 2.0.9 1.5.0 - 2.11.0 + 2.14.0 1.10.0 3.12.0 2.6.2.Final From cfc3f84593da8d8805583bc4d003661ff02ba85c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:20:40 +0000 Subject: [PATCH 7/7] chore(deps): Bump tempfile from 3.10.0 to 3.13.0 (#18772) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 63 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ca90675afc0d..d6801d7f3f40a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1059,7 +1059,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 0.38.31", + "rustix 0.38.37", "slab", "tracing", "windows-sys 0.52.0", @@ -2421,7 +2421,7 @@ dependencies = [ "io-lifetimes 2.0.3", "ipnet", "maybe-owned", - "rustix 0.38.31", + "rustix 0.38.37", "windows-sys 0.52.0", "winx", ] @@ -2445,7 +2445,7 @@ dependencies = [ "cap-primitives", "io-extras", "io-lifetimes 2.0.3", - "rustix 0.38.31", + "rustix 0.38.37", ] [[package]] @@ -2458,7 +2458,7 @@ dependencies = [ "cap-primitives", "iana-time-zone", "once_cell", - "rustix 0.38.31", + "rustix 0.38.37", "winx", ] @@ -4906,9 +4906,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fd-lock" @@ -4917,7 +4917,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93f7a0db71c99f68398f80653ed05afb0b00e062e1a20c7ff849c4edfabbbcc" dependencies = [ "cfg-if", - "rustix 0.38.31", + "rustix 0.38.37", "windows-sys 0.52.0", ] @@ -5299,7 +5299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb" dependencies = [ "io-lifetimes 2.0.3", - "rustix 0.38.31", + "rustix 0.38.37", "windows-sys 0.52.0", ] @@ -5319,7 +5319,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c6b3bd49c37d2aa3f3f2220233b29a7cd23f79d1fe70e5337d25fb390793de" dependencies = [ - "rustix 0.38.31", + "rustix 0.38.37", "windows-sys 0.52.0", ] @@ -6655,7 +6655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.9", - "rustix 0.38.31", + "rustix 0.38.37", "windows-sys 0.48.0", ] @@ -6959,9 +6959,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libflate" @@ -7094,9 +7094,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "local_stats_alloc" @@ -7454,7 +7454,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 0.38.31", + "rustix 0.38.37", ] [[package]] @@ -9122,7 +9122,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.31", + "rustix 0.38.37", "tracing", "windows-sys 0.52.0", ] @@ -9442,7 +9442,7 @@ dependencies = [ "bitflags 2.6.0", "hex", "procfs-core", - "rustix 0.38.31", + "rustix 0.38.37", ] [[package]] @@ -12436,15 +12436,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", "itoa", "libc", - "linux-raw-sys 0.4.12", + "linux-raw-sys 0.4.14", "once_cell", "windows-sys 0.52.0", ] @@ -14231,7 +14231,7 @@ dependencies = [ "cap-std", "fd-lock", "io-lifetimes 2.0.3", - "rustix 0.38.31", + "rustix 0.38.37", "windows-sys 0.52.0", "winx", ] @@ -14262,14 +14262,15 @@ checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "tempfile" -version = "3.10.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", - "rustix 0.38.31", - "windows-sys 0.52.0", + "once_cell", + "rustix 0.38.37", + "windows-sys 0.59.0", ] [[package]] @@ -15475,7 +15476,7 @@ dependencies = [ "io-lifetimes 2.0.3", "log", "once_cell", - "rustix 0.38.31", + "rustix 0.38.37", "system-interface", "thiserror", "tracing", @@ -15641,7 +15642,7 @@ dependencies = [ "postcard", "psm", "rayon", - "rustix 0.38.31", + "rustix 0.38.37", "semver 1.0.18", "serde", "serde_derive", @@ -15687,7 +15688,7 @@ dependencies = [ "directories-next", "log", "postcard", - "rustix 0.38.31", + "rustix 0.38.37", "serde", "serde_derive", "sha2", @@ -15775,7 +15776,7 @@ dependencies = [ "anyhow", "cc", "cfg-if", - "rustix 0.38.31", + "rustix 0.38.37", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", "windows-sys 0.52.0", @@ -15789,7 +15790,7 @@ checksum = "3eb2b7545bf13c125d007fd1808cf4c6fb4688258e47f7a3ea96ffc04d173a15" dependencies = [ "object 0.36.4", "once_cell", - "rustix 0.38.31", + "rustix 0.38.37", "wasmtime-versioned-export-macros", ] @@ -15945,7 +15946,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.31", + "rustix 0.38.37", ] [[package]]