From b44b5b8e9f1868df2662022fa4e53330105e88aa Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Wed, 3 Jul 2024 18:35:52 +0000 Subject: [PATCH] Upgrade the deltalake-azure dependency to use the right version spec --- Cargo.lock | 681 +++++++------------------------ Cargo.toml | 23 +- tests/emails_azure_blob_tests.rs | 8 +- 3 files changed, 167 insertions(+), 545 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b23a4a3..4bad2e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -51,7 +51,7 @@ dependencies = [ "getrandom 0.2.15", "once_cell", "version_check", - "zerocopy 0.7.34", + "zerocopy 0.7.35", ] [[package]] @@ -191,61 +191,25 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "arrow" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" -dependencies = [ - "arrow-arith 51.0.0", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-csv 51.0.0", - "arrow-data 51.0.0", - "arrow-ipc 51.0.0", - "arrow-json 51.0.0", - "arrow-ord 51.0.0", - "arrow-row 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "arrow-string 51.0.0", -] - [[package]] name = "arrow" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ - "arrow-arith 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-csv 52.0.0", - "arrow-data 52.0.0", - "arrow-ipc 52.0.0", - "arrow-json 52.0.0", - "arrow-ord 52.0.0", - "arrow-row 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", - "arrow-string 52.0.0", -] - -[[package]] -name = "arrow-arith" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "half", - "num", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -254,29 +218,12 @@ version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "chrono", - "half", - "num", -] - -[[package]] -name = "arrow-array" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" -dependencies = [ - "ahash", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", - "chrono-tz 0.8.6", "half", - "hashbrown", "num", ] @@ -287,27 +234,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", - "chrono-tz 0.9.0", + "chrono-tz", "half", "hashbrown", "num", ] -[[package]] -name = "arrow-buffer" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" -dependencies = [ - "bytes", - "half", - "num", -] - [[package]] name = "arrow-buffer" version = "52.0.0" @@ -319,37 +255,17 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-cast" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "atoi", - "base64 0.22.1", - "chrono", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64 0.22.1", "chrono", @@ -360,36 +276,17 @@ dependencies = [ "ryu", ] -[[package]] -name = "arrow-csv" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "csv", - "csv-core", - "lazy_static", - "lexical-core", - "regex", -] - [[package]] name = "arrow-csv" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "csv", "csv-core", @@ -398,90 +295,44 @@ dependencies = [ "regex", ] -[[package]] -name = "arrow-data" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" -dependencies = [ - "arrow-buffer 51.0.0", - "arrow-schema 51.0.0", - "half", - "num", -] - [[package]] name = "arrow-data" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ - "arrow-buffer 52.0.0", - "arrow-schema 52.0.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] -[[package]] -name = "arrow-ipc" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "flatbuffers 23.5.26", -] - [[package]] name = "arrow-ipc" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "flatbuffers 24.3.25", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", "lz4_flex", ] -[[package]] -name = "arrow-json" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "chrono", - "half", - "indexmap", - "lexical-core", - "num", - "serde", - "serde_json", -] - [[package]] name = "arrow-json" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -491,51 +342,21 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-ord" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "half", - "num", -] - [[package]] name = "arrow-ord" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "half", "num", ] -[[package]] -name = "arrow-row" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" -dependencies = [ - "ahash", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "half", - "hashbrown", -] - [[package]] name = "arrow-row" version = "52.0.0" @@ -543,23 +364,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", "hashbrown", ] -[[package]] -name = "arrow-schema" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" -dependencies = [ - "serde", -] - [[package]] name = "arrow-schema" version = "52.0.0" @@ -569,20 +381,6 @@ dependencies = [ "serde", ] -[[package]] -name = "arrow-select" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" -dependencies = [ - "ahash", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "num", -] - [[package]] name = "arrow-select" version = "52.0.0" @@ -590,41 +388,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] -[[package]] -name = "arrow-string" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" -dependencies = [ - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-data 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "memchr", - "num", - "regex", - "regex-syntax", -] - [[package]] name = "arrow-string" version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-data 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num", "regex", @@ -1225,17 +1006,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "brotli" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 2.5.1", -] - [[package]] name = "brotli" version = "6.0.0" @@ -1244,17 +1014,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 4.0.1", -] - -[[package]] -name = "brotli-decompressor" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -1354,17 +1114,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "chrono-tz" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" -dependencies = [ - "chrono", - "chrono-tz-build 0.2.1", - "phf 0.11.2", -] - [[package]] name = "chrono-tz" version = "0.9.0" @@ -1372,21 +1121,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono", - "chrono-tz-build 0.3.0", + "chrono-tz-build", "phf 0.11.2", ] -[[package]] -name = "chrono-tz-build" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" -dependencies = [ - "parse-zoneinfo", - "phf 0.11.2", - "phf_codegen 0.11.2", -] - [[package]] name = "chrono-tz-build" version = "0.3.0" @@ -1602,10 +1340,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" dependencies = [ "ahash", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-ipc 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", "async-compression", "async-trait", "bytes", @@ -1635,7 +1373,7 @@ dependencies = [ "num_cpus", "object_store 0.10.1", "parking_lot", - "parquet 52.0.0", + "parquet", "paste", "pin-project-lite", "rand 0.8.5", @@ -1656,10 +1394,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" dependencies = [ "ahash", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", "chrono", "half", "hashbrown", @@ -1667,7 +1405,7 @@ dependencies = [ "libc", "num_cpus", "object_store 0.10.1", - "parquet 52.0.0", + "parquet", "sqlparser", ] @@ -1686,7 +1424,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" dependencies = [ - "arrow 52.0.0", + "arrow", "chrono", "dashmap", "datafusion-common", @@ -1708,9 +1446,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" dependencies = [ "ahash", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", @@ -1726,7 +1464,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" dependencies = [ - "arrow 52.0.0", + "arrow", "base64 0.22.1", "blake2", "blake3", @@ -1754,8 +1492,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" dependencies = [ "ahash", - "arrow 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1771,11 +1509,11 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" dependencies = [ - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1791,7 +1529,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" dependencies = [ - "arrow 52.0.0", + "arrow", "async-trait", "chrono", "datafusion-common", @@ -1811,12 +1549,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" dependencies = [ "ahash", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", - "arrow-string 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", "base64 0.22.1", "chrono", "datafusion-common", @@ -1841,7 +1579,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" dependencies = [ - "arrow 52.0.0", + "arrow", "datafusion-common", "datafusion-expr", "rand 0.8.5", @@ -1854,11 +1592,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" dependencies = [ "ahash", - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -1887,7 +1625,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db8eb8d706c73f01c0e2630c64ffc61c33831b064a813ec08a3e094dc3190c0f" dependencies = [ - "arrow 52.0.0", + "arrow", "chrono", "datafusion", "datafusion-common", @@ -1903,7 +1641,7 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde7115772d326eeb78a1c77c974e7753b1538e1747675cb6b428058b654b31c" dependencies = [ - "arrow 52.0.0", + "arrow", "chrono", "datafusion-common", "object_store 0.9.1", @@ -1916,9 +1654,9 @@ version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" dependencies = [ - "arrow 52.0.0", - "arrow-array 52.0.0", - "arrow-schema 52.0.0", + "arrow", + "arrow-array", + "arrow-schema", "datafusion-common", "datafusion-expr", "log", @@ -1943,12 +1681,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1ddfe35af3696786ab5f23cd995df33a66f6cff272ac1f85e09c1a6316acd4c" dependencies = [ - "arrow-arith 52.0.0", - "arrow-array 52.0.0", - "arrow-json 52.0.0", - "arrow-ord 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-arith", + "arrow-array", + "arrow-json", + "arrow-ord", + "arrow-schema", + "arrow-select", "bytes", "chrono", "delta_kernel_derive", @@ -1957,7 +1695,7 @@ dependencies = [ "indexmap", "itertools 0.13.0", "lazy_static", - "parquet 52.0.0", + "parquet", "roaring", "rustc_version", "serde", @@ -1995,7 +1733,7 @@ dependencies = [ "aws-smithy-runtime-api", "backoff", "bytes", - "deltalake-core 0.18.1", + "deltalake-core", "futures", "lazy_static", "maplit", @@ -2010,75 +1748,21 @@ dependencies = [ [[package]] name = "deltalake-azure" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e074599ebb06706867093e06f29ca4eb77a310a8ca24c83066063d4062acbbfe" +checksum = "299f7bad095ba316d7c009bba4c423761393b3499b9ed1cca2b027ab6b201e53" dependencies = [ "async-trait", "bytes", - "deltalake-core 0.17.3", + "deltalake-core", "futures", "lazy_static", - "object_store 0.9.1", - "regex", - "thiserror", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "deltalake-core" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8dc1bcd91be689ee7f6ce363798dc2e9b6954be9d597f884de11ba27b33add" -dependencies = [ - "arrow 51.0.0", - "arrow-arith 51.0.0", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-ipc 51.0.0", - "arrow-json 51.0.0", - "arrow-ord 51.0.0", - "arrow-row 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "async-trait", - "bytes", - "cfg-if", - "chrono", - "dashmap", - "either", - "errno", - "fix-hidden-lifetime-bug", - "futures", - "hashbrown", - "indexmap", - "itertools 0.12.1", - "lazy_static", - "libc", - "maplit", - "num-bigint", - "num-traits", - "num_cpus", - "object_store 0.9.1", - "once_cell", - "parking_lot", - "parquet 51.0.0", - "percent-encoding", - "pin-project-lite", - "rand 0.8.5", + "object_store 0.10.1", "regex", - "roaring", - "serde", - "serde_json", "thiserror", "tokio", "tracing", "url", - "uuid 1.9.1", - "z85", ] [[package]] @@ -2087,17 +1771,17 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0068bd92347795c1c5e3b4ef7c5489de289ebe78cc9ab6667c0fbab539da3d7c" dependencies = [ - "arrow 52.0.0", - "arrow-arith 52.0.0", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-ipc 52.0.0", - "arrow-json 52.0.0", - "arrow-ord 52.0.0", - "arrow-row 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", "async-trait", "bytes", "cfg-if", @@ -2128,7 +1812,7 @@ dependencies = [ "object_store 0.10.1", "once_cell", "parking_lot", - "parquet 52.0.0", + "parquet", "percent-encoding", "pin-project-lite", "rand 0.8.5", @@ -2368,16 +2052,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" -[[package]] -name = "flatbuffers" -version = "23.5.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" -dependencies = [ - "bitflags 1.3.2", - "rustc_version", -] - [[package]] name = "flatbuffers" version = "24.3.25" @@ -2882,7 +2556,7 @@ dependencies = [ "hyper 1.4.0", "hyper-util", "rustls 0.23.10", - "rustls-native-certs 0.7.0", + "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -3090,7 +2764,7 @@ dependencies = [ "clap", "deltalake-aws", "deltalake-azure", - "deltalake-core 0.18.1", + "deltalake-core", "dipstick", "dynamodb_lock", "env_logger", @@ -3514,21 +3188,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", - "base64 0.21.7", "bytes", "chrono", "futures", "humantime", - "hyper 0.14.29", "itertools 0.12.1", "parking_lot", "percent-encoding", - "quick-xml", - "rand 0.8.5", - "reqwest 0.11.27", - "ring 0.17.8", - "serde", - "serde_json", "snafu", "tokio", "tracing", @@ -3676,41 +3342,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "parquet" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" -dependencies = [ - "ahash", - "arrow-array 51.0.0", - "arrow-buffer 51.0.0", - "arrow-cast 51.0.0", - "arrow-data 51.0.0", - "arrow-ipc 51.0.0", - "arrow-schema 51.0.0", - "arrow-select 51.0.0", - "base64 0.22.1", - "brotli 3.5.0", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown", - "lz4_flex", - "num", - "num-bigint", - "object_store 0.9.1", - "paste", - "seq-macro", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", -] - [[package]] name = "parquet" version = "52.0.0" @@ -3718,15 +3349,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash", - "arrow-array 52.0.0", - "arrow-buffer 52.0.0", - "arrow-cast 52.0.0", - "arrow-data 52.0.0", - "arrow-ipc 52.0.0", - "arrow-schema 52.0.0", - "arrow-select 52.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64 0.22.1", - "brotli 6.0.0", + "brotli", "bytes", "chrono", "flate2", @@ -3949,9 +3580,9 @@ dependencies = [ [[package]] name = "psl" -version = "2.1.49" +version = "2.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11928d9b009645f617a3ff75bd992ff150dd239b1ad8e2949d36c0dc998438a" +checksum = "20f2890e9f6f068d927643e8d547931b537f21fd298e197c4837f1265b11e560" dependencies = [ "psl-types", ] @@ -4215,7 +3846,6 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.29", - "hyper-rustls 0.24.2", "hyper-tls", "ipnet", "js-sys", @@ -4225,8 +3855,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -4235,7 +3863,6 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", "tokio-util 0.7.11", "tower-service", "url", @@ -4272,7 +3899,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.10", - "rustls-native-certs 0.7.0", + "rustls-native-certs 0.7.1", "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", @@ -4329,9 +3956,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" [[package]] name = "roaring" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7699249cc2c7d71939f30868f47e9d7add0bdc030d90ee10bfd16887ff8bb1c8" +checksum = "8f4b84ba6e838ceb47b41de5194a60244fac43d9fe03b71dbe8c5a201081d6d1" dependencies = [ "bytemuck", "byteorder", @@ -4501,7 +4128,7 @@ dependencies = [ "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] @@ -4532,9 +4159,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.2", @@ -4580,9 +4207,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -6015,11 +5642,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive 0.7.34", + "zerocopy-derive 0.7.35", ] [[package]] @@ -6035,9 +5662,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 6e0ca93..9f34759 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,27 +35,19 @@ url = "2.3" # datafusion feature is required for writer version 2 deltalake-core = { version = "~0.18.1", features = ["json", "datafusion"]} deltalake-aws = { version = "~0.1.2", optional = true } -deltalake-azure = { version = "~0.1.2", optional = true } +deltalake-azure = { version = "~0.1.3", optional = true } -# s3 feature enabled +# s3 feature enabled, helps for locking interactions with DLQ dynamodb_lock = { version = "0.6.0", optional = true } # sentry sentry = { version = "0.23.0", optional = true } -# azure feature enabled, mostly used for tests -azure_core = { version = "0.18.0", optional = true } -azure_storage = { version = "0.18.0", optional = true } -azure_storage_blobs = { version = "0.18.0", optional = true } - [features] default = [] sentry-ext = ["sentry"] dynamic-linking = [ "rdkafka/dynamic-linking" ] azure = [ "deltalake-azure", - "azure_core", - "azure_storage", - "azure_storage_blobs" ] s3 = [ "deltalake-aws", @@ -63,13 +55,16 @@ s3 = [ ] [dev-dependencies] -utime = "0.3" -serial_test = "*" -tempfile = "3" -time = "0.3.20" +azure_core = { version = "0.18.0" } +azure_storage = { version = "0.18.0" } +azure_storage_blobs = { version = "0.18.0" } rusoto_core = { version = "0.47", default-features = false, features = ["rustls"]} rusoto_credential = { version = "0.47"} rusoto_s3 = { version = "0.47", default-features = false, features = ["rustls"]} +serial_test = "*" +tempfile = "3" +time = "0.3.20" +utime = "0.3" [profile.release] lto = true diff --git a/tests/emails_azure_blob_tests.rs b/tests/emails_azure_blob_tests.rs index d85f111..1d8efb0 100644 --- a/tests/emails_azure_blob_tests.rs +++ b/tests/emails_azure_blob_tests.rs @@ -20,18 +20,18 @@ use azure_storage::{prelude::BlobSasPermissions, shared_access_signature::SasPro #[tokio::test(flavor = "multi_thread")] #[serial] async fn when_both_workers_started_simultaneously_azure() { - run_emails_s3_tests(false).await; + run_emails_azure_tests(false).await; } #[tokio::test(flavor = "multi_thread")] #[serial] async fn when_rebalance_happens_azure() { - run_emails_s3_tests(true).await; + run_emails_azure_tests(true).await; } -async fn run_emails_s3_tests(initiate_rebalance: bool) { - deltalake_azure::register_handlers(None); +async fn run_emails_azure_tests(initiate_rebalance: bool) { helpers::init_logger(); + deltalake_azure::register_handlers(None); let topic = format!("emails_azure-{}", Uuid::new_v4()); let table = prepare_table(&topic).await; let options = create_options();