Merge branch 'main' into xxh/stream-subscription

risingwavelabs · Jun 26, 2024 · bbba5e0 · bbba5e0
2 parents 6d7460d + 4e22820
commit bbba5e0
Show file tree

Hide file tree

Showing 7 changed files with 662 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -138,6 +138,10 @@ arrow-array-iceberg = { package = "arrow-array", version = "52" }
 arrow-schema-iceberg = { package = "arrow-schema", version = "52" }
 arrow-buffer-iceberg = { package = "arrow-buffer", version = "52" }
 arrow-cast-iceberg = { package = "arrow-cast", version = "52" }
+# TODO
+# After apache/iceberg-rust#411 is merged, we move to the upstream version.
+iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "0c6e133e6f4655ff9ce4ad57b577dc7f692dd902" }
+iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "0c6e133e6f4655ff9ce4ad57b577dc7f692dd902" }
 arrow-array = "50"
 arrow-arith = "50"
 arrow-cast = "50"

diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml
@@ -64,6 +64,8 @@ google-cloud-gax = "0.17.0"
 google-cloud-googleapis = { version = "0.13", features = ["pubsub", "bigquery"] }
 google-cloud-pubsub = "0.25"
 http = "0.2"
+iceberg = { workspace = true }
+iceberg-catalog-rest = { workspace = true }
 icelake = { workspace = true }
 indexmap = { version = "2.2.6", features = ["serde"] }
 itertools = { workspace = true }
@@ -90,6 +92,7 @@ opendal = { version = "0.47", features = [
 ] }
 openssl = "0.10"
 parking_lot = { workspace = true }
+parquet = { workspace = true }
 paste = "1"
 pg_bigdecimal = { git = "https://github.com/risingwavelabs/rust-pg_bigdecimal", rev = "0b7893d88894ca082b4525f94f812da034486f7c" }
 postgres-openssl = "0.5.0"
@@ -160,6 +163,7 @@ tokio-stream = "0.1"
 tokio-util = { version = "0.7", features = ["codec", "io"] }
 tonic = { workspace = true }
 tracing = "0.1"
+typed-builder = "^0.18"
 url = "2"
 urlencoding = "2"
 uuid = { version = "1", features = ["v4", "fast-rng"] }

diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs
@@ -57,6 +57,7 @@ def_anyhow_newtype! {
     async_nats::jetstream::context::CreateStreamError => "Nats error",
     async_nats::jetstream::stream::ConsumerError => "Nats error",
     icelake::Error => "Iceberg error",
+    iceberg::Error => "IcebergV2 error",
     redis::RedisError => "Redis error",
     arrow_schema::ArrowError => "Arrow error",
     arrow_schema_iceberg::ArrowError => "Arrow error",