test: add codec integration tests

risingwavelabs · Jul 2, 2024 · ab6c3aa · ab6c3aa
1 parent 5bfa202
commit ab6c3aa
Show file tree

Hide file tree

Showing 13 changed files with 532 additions and 323 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/docs/developer-guide.md b/docs/developer-guide.md
@@ -35,7 +35,7 @@ http://ecotrust-canada.github.io/markdown-toc/
   * [Logging](#logging)
 - [Test your code changes](#test-your-code-changes)
   * [Lint](#lint)
-  * [Unit tests](#unit-tests)
+  * [Unit and integration tests](#unit-and-integration-tests)
   * [Planner tests](#planner-tests)
   * [End-to-end tests](#end-to-end-tests)
   * [Fuzzing tests](#fuzzing-tests)
@@ -315,14 +315,21 @@ RisingWave requires all code to pass fmt, clippy, sort and hakari checks. Run th
 
 There are also some miscellaneous checks. See `ci/scripts/misc-check.sh`.
 
-### Unit tests
+### Unit and integration tests
 
 RiseDev runs unit tests with cargo-nextest. To run unit tests:
 
 ```shell
 ./risedev test          # Run unit tests
 ```
 
+Some ideas and caveats for writing tests:
+- Use [expect_test](https://github.com/rust-analyzer/expect-test) to write data driven tests that can automatically update results.
+- It's recommended to write new tests as *integration tests* (i.e. in `tests/` directory) instead of *unit tests* (i.e. in `src/` directory).
+
+  Besides, put integration tests under `tests/integration_tests/*.rs`, instead of `tests/*.rs`. See [Delete Cargo Integration Tests](https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html) and [#9878](https://github.com/risingwavelabs/risingwave/issues/9878), for more details.
+
+You might want to read [How to Test](https://matklad.github.io/2021/05/31/how-to-test.html) for more good ideas on testing.
 
 ### Planner tests
 

diff --git a/src/connector/codec/Cargo.toml b/src/connector/codec/Cargo.toml
@@ -35,8 +35,16 @@ thiserror-ext = { workspace = true }
 time = "0.3.30"
 tracing = "0.1"
 
+[dev-dependencies]
+expect-test = "1"
+hex = "0.4"
+
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../../workspace-hack" }
 
 [lints]
 workspace = true
+
+[lib]
+test = false
+doctest = false
diff --git a/src/connector/codec/src/decoder/avro/schema.rs b/src/connector/codec/src/decoder/avro/schema.rs
@@ -74,6 +74,8 @@ impl MapHandling {
 
 /// This function expects resolved schema (no `Ref`).
 /// FIXME: require passing resolved schema here.
+/// TODO: change `map_handling` to some `Config`, and also unify debezium.
+/// TODO: use `ColumnDesc` in common instead of PB.
 pub fn avro_schema_to_column_descs(
     schema: &Schema,
     map_handling: Option<MapHandling>,

diff --git a/src/connector/codec/src/decoder/mod.rs b/src/connector/codec/src/decoder/mod.rs
@@ -44,7 +44,7 @@ pub enum AccessError {
 
 pub type AccessResult<T = Datum> = std::result::Result<T, AccessError>;
 
-/// Access to a field in the data structure.
+/// Access to a field in the data structure. Created by `AccessBuilder`.
 pub trait Access {
     /// Accesses `path` in the data structure (*parsed* Avro/JSON/Protobuf data),
     /// and then converts it to RisingWave `Datum`.

diff --git a/src/connector/codec/src/lib.rs b/src/connector/codec/src/lib.rs
@@ -44,6 +44,7 @@
 pub mod decoder;
 
 pub use apache_avro::schema::Schema as AvroSchema;
+pub use apache_avro::types::{Value as AvroValue, ValueKind as AvroValueKind};
 pub use risingwave_pb::plan_common::ColumnDesc;
 pub struct JsonSchema(pub serde_json::Value);
 impl JsonSchema {