Skip to content

Commit

Permalink
Disable scheam inference when schema evolution is disabled
Browse files Browse the repository at this point in the history
This will ensure the non-evolution case stands relatively speedy!

Sponsored-by: Raft LLC
  • Loading branch information
rtyler committed Jan 9, 2024
1 parent e4c0f1e commit 50e81da
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 61 deletions.
126 changes: 78 additions & 48 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 3 additions & 6 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,17 @@ serde_json = "1"
strum_macros = "0.20"
thiserror = "1"
tokio = { version = "1", features = ["full"] }
tokio-util = "0.6.3"
tokio-util = "0.7.10"
uuid = { version = "1.0", features = ["serde", "v4"] }
url = "2.3"

#deltalake = { version = "0.16.5", features = ["arrow", "json", "parquet"], optional = true }
deltalake-core = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["json"]}
deltalake-core = { git = "https://github.com/rtyler/delta-rs", branch = "createdTime_is_optional", features = ["json"]}
deltalake-aws = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true }
deltalake-azure = { git = "https://github.com/delta-io/delta-rs", branch = "main", optional = true }

# s3 feature enabled
dynamodb_lock = { version = "0.6.0", optional = true }
rusoto_core = { version = "0.47", default-features = false, features = ["rustls"], optional = true }
rusoto_credential = { version = "0.47", optional = true }

# sentry
sentry = { version = "0.23.0", optional = true }
Expand Down Expand Up @@ -68,15 +66,14 @@ azure = [
s3 = [
"deltalake-aws",
"dynamodb_lock",
"rusoto_core",
"rusoto_credential",
]

[dev-dependencies]
utime = "0.3"
serial_test = "*"
tempfile = "3"
time = "0.3.20"
rusoto_core = { version = "0.47", default-features = false, features = ["rustls"]}
rusoto_s3 = { version = "0.47", default-features = false, features = ["rustls"]}

[profile.release]
Expand Down
2 changes: 1 addition & 1 deletion src/dead_letters.rs
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ impl DeltaSinkDeadLetterQueue {
dynamo_lock_options::DYNAMO_LOCK_PARTITION_KEY_VALUE.to_string() => std::env::var(env_vars::DEAD_LETTER_DYNAMO_LOCK_PARTITION_KEY_VALUE)
.unwrap_or_else(|_| "kafka_delta_ingest-dead_letters".to_string()),
};
#[cfg(all(feature = "azure", not(feature="s3")))]
#[cfg(all(feature = "azure", not(feature = "s3")))]
let opts = HashMap::default();

let table = crate::delta_helpers::load_table(table_uri, opts.clone()).await?;
Expand Down
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#![deny(warnings)]
#![deny(missing_docs)]
#![allow(unused)]

#[macro_use]
extern crate lazy_static;
Expand Down
Loading

0 comments on commit 50e81da

Please sign in to comment.