From 99609b9882a49601bb9aa141d339c4584f50cf12 Mon Sep 17 00:00:00 2001 From: MrCroxx Date: Tue, 7 Nov 2023 17:16:05 +0800 Subject: [PATCH 1/3] refactor: bump aws sdk, use rustls in s3 client Signed-off-by: MrCroxx --- Cargo.lock | 633 +++++++++++------- Cargo.toml | 31 +- src/bench/Cargo.toml | 1 + src/bench/s3_bench/main.rs | 2 +- src/connector/Cargo.toml | 2 + src/connector/src/aws_auth.rs | 3 +- .../src/source/filesystem/s3/source/reader.rs | 17 +- .../src/source/kinesis/enumerator/client.rs | 2 +- .../src/source/kinesis/source/message.rs | 6 +- .../src/source/kinesis/source/reader.rs | 2 +- src/meta/src/rpc/cloud_provider.rs | 26 +- src/object_store/Cargo.toml | 7 +- src/object_store/src/object/error.rs | 16 +- src/object_store/src/object/s3.rs | 104 +-- src/tests/simulation/Cargo.toml | 2 +- src/workspace-hack/Cargo.toml | 10 +- 16 files changed, 500 insertions(+), 364 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcac410f23035..ed03deac0f3fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -511,11 +511,11 @@ dependencies = [ "once_cell", "rand", "regex", - "ring", - "rustls 0.21.7", + "ring 0.16.20", + "rustls", "rustls-native-certs", "rustls-pemfile", - "rustls-webpki 0.101.4", + "rustls-webpki 0.101.7", "serde", "serde_json", "serde_nanos", @@ -524,7 +524,7 @@ dependencies = [ "time", "tokio", "tokio-retry", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "url", ] @@ -685,148 +685,144 @@ dependencies = [ [[package]] name = "aws-config" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" +checksum = "7489a72550db3712fe3a0a92068f832d6270ff82f518b84a800af131f99570d7" dependencies = [ "aws-credential-types", "aws-http", + "aws-runtime", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", - "fastrand 1.9.0", + "fastrand 2.0.0", "http", "hyper", "time", "tokio", - "tower", "tracing", ] [[package]] name = "aws-credential-types" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" +checksum = "80009846d61a0a4f9070d789cf0e64db284cba6984fae3871050d044e6569cd2" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand 1.9.0", - "tokio", - "tracing", "zeroize", ] [[package]] -name = "aws-endpoint" -version = "0.55.3" +name = "aws-http" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" +checksum = "1e65730b741a5f6422fd338bf6f76b7956b090affeaa045e78fca8c4186e0fd5" dependencies = [ "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes", "http", - "regex", + "http-body", + "pin-project-lite", "tracing", ] [[package]] -name = "aws-http" -version = "0.55.3" +name = "aws-runtime" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" +checksum = "1d2414b96071ae840b97c0cc1d44b248d5607d648593cdf474f3fb5465572898" dependencies = [ "aws-credential-types", + "aws-http", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes", + "fastrand 2.0.0", "http", - "http-body", - "lazy_static", "percent-encoding", - "pin-project-lite", "tracing", + "uuid", ] [[package]] name = "aws-sdk-ec2" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab2493c5857725eeafe12ec66ba4ce6feb3355e3af6828d9ef28d6152972a27" +checksum = "7d3492d8b17bc47532abee2d83a3f861997a7017ca548abb0ab4a26d5e6a691b" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", - "bytes", - "fastrand 1.9.0", + "fastrand 2.0.0", "http", "regex", - "tokio-stream", - "tower", "tracing", ] [[package]] name = "aws-sdk-kinesis" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca88060b315da80279486d079a2c5c27891fc60a7e770526e50ad5d98551f650" +checksum = "5acef0ae3beb145ba41e640fe33b2b5ac021218ee0b42199779393d2b7e4e2cc" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "http", "regex", - "tokio-stream", - "tower", "tracing", ] [[package]] name = "aws-sdk-s3" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba197193cbb4bcb6aad8d99796b2291f36fa89562ded5d4501363055b0de89f" +checksum = "84022763485483ea17d417f9832d5da198bc36829b59f086c0d35ecd2ce59991" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", - "aws-smithy-client", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -836,91 +832,76 @@ dependencies = [ "once_cell", "percent-encoding", "regex", - "tokio-stream", - "tower", "tracing", "url", ] [[package]] name = "aws-sdk-sts" -version = "0.28.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" +checksum = "51b1a8ae5c7098502a3e6d4130dbee1e1d3fcb8dc5d65cecab39e01d595f90f6" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", - "bytes", "http", "regex", - "tower", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", - "http", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +checksum = "3347c738e0a8449020877d319cda56da74d6e8aba9fff210720fac66cae3c7f4" dependencies = [ + "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", "bytes", "form_urlencoded", "hex", "hmac", "http", + "num-bigint", "once_cell", + "p256 0.11.1", "percent-encoding", "regex", + "ring 0.17.5", "sha2", "time", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-async" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" +checksum = "b4b65a284265d3eec6cc9f1daef2d0cc3b78684b712cb6c7f1d0f665456b7604" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "aws-smithy-checksums" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ed8b96d95402f3f6b8b57eb4e0e45ee365f78b1a924faf20ff6e97abf1eae6" +checksum = "d40f1d5a222ba11ac7d6b20f3668ae282970e50615fa5ee1dd8ac8180c0c1803" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -937,36 +918,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 1.9.0", - "http", - "http-body", - "hyper", - "hyper-rustls 0.23.2", - "hyper-tls", - "lazy_static", - "pin-project-lite", - "rustls 0.20.9", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460c8da5110835e3d9a717c61f5556b20d03c32a1dec57f8fc559b360f733bb8" +checksum = "b16e7ecebc2b083a1b138868a46a343204a6097f343c4830a8b22b3a0d30013e" dependencies = [ "aws-smithy-types", "bytes", @@ -975,94 +931,124 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +checksum = "715aeb61fb743848d5d398ce6fb1259f5eba5e13dceec5d5064cada1a181d38d" dependencies = [ "aws-smithy-eventstream", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http", "http-body", - "hyper", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "tokio", - "tokio-util", "tracing", ] [[package]] -name = "aws-smithy-http-tower" -version = "0.55.3" +name = "aws-smithy-json" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" +checksum = "de21d368dcd5cab17033406ea6e7351b091164b208381de837510bd7558c0f30" dependencies = [ - "aws-smithy-http", "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", ] [[package]] -name = "aws-smithy-json" -version = "0.55.3" +name = "aws-smithy-query" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +checksum = "9e5ace389c7e4def130bff7275647481c8d49b867909ca61d5dc9a809b3632f3" dependencies = [ "aws-smithy-types", + "urlencoding", ] [[package]] -name = "aws-smithy-query" -version = "0.55.3" +name = "aws-smithy-runtime" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +checksum = "fb4395310662d10f1847324af5fe43e621922cba03b1aa6d26c21096e18a4e79" dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", - "urlencoding", + "bytes", + "fastrand 2.0.0", + "http", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e27594c06f5b36e97d18ef26ed693f1d4c7167b9bbb544b3a9bb653f9f7035" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", ] [[package]] name = "aws-smithy-types" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +checksum = "5d36f1723ed61e82094498e7283510fe21484b73c215c33874c81a84411b5bdc" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", + "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" +checksum = "68225c8d3e3e6c565a3cf764aa82440837ef15c33d1dd7205e15715444e4b4ad" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.3" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" +checksum = "acdc27aac60f715bab25f5d758ba5651b80aae791c48e9871ffe298683f00a2b" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "http", "rustc_version", @@ -1158,6 +1144,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base16ct" version = "0.2.0" @@ -2172,6 +2164,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.3" @@ -2451,6 +2455,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der" version = "0.7.8" @@ -2633,18 +2647,30 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49457524c7e65648794c98283282a0b7c73b10018e7091f1cdcfff314fd7ae59" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + [[package]] name = "ecdsa" version = "0.16.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4b1e0c257a9e9f25f90ff76d7a68360ed497ee519c8e428d1825ef0000799d4" dependencies = [ - "der", + "der 0.7.8", "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "elliptic-curve 0.13.6", + "rfc6979 0.4.0", + "signature 2.0.0", + "spki 0.7.2", ] [[package]] @@ -2653,8 +2679,8 @@ version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.0.0", ] [[package]] @@ -2667,7 +2693,7 @@ dependencies = [ "ed25519", "serde", "sha2", - "signature", + "signature 2.0.0", "zeroize", ] @@ -2692,23 +2718,43 @@ dependencies = [ "serde", ] +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", + "digest", + "ff 0.12.1", + "generic-array", + "group 0.12.1", + "pkcs8 0.9.0", + "rand_core", + "sec1 0.3.0", + "subtle", + "zeroize", +] + [[package]] name = "elliptic-curve" version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d97ca172ae9dc9f9b779a6e3a65d308f2af74e5b8c921299075bdb4a0370e914" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.3", "digest", - "ff", + "ff 0.13.0", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core", - "sec1", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -2964,6 +3010,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "ff" version = "0.13.0" @@ -3462,7 +3518,7 @@ dependencies = [ "async-trait", "dyn-clone", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls", "log", "reqwest", "serde", @@ -3501,8 +3557,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" -source = "git+https://github.com/madsim-rs/getrandom.git?rev=8daf97e#8daf97e4142635fe28543b2db9022f5e2544bb5c" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "js-sys", @@ -3640,13 +3697,24 @@ dependencies = [ "smallvec", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff 0.12.1", + "rand_core", + "subtle", +] + [[package]] name = "group" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.0", "rand_core", "subtle", ] @@ -3888,33 +3956,19 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.9", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", -] - -[[package]] -name = "hyper-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", "hyper", "log", - "rustls 0.21.7", + "rustls", "rustls-native-certs", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", + "webpki-roots 0.25.2", ] [[package]] @@ -4279,7 +4333,7 @@ checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.4", "pem 1.1.1", - "ring", + "ring 0.16.20", "serde", "serde_json", "simple_asn1", @@ -4642,11 +4696,12 @@ dependencies = [ [[package]] name = "madsim-aws-sdk-s3" version = "0.2.25+0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ab622f10406b583fb75af6f94811d5ae003c1197b9fbaec011ff1d323fc9b6" +source = "git+https://github.com/mrcroxx/madsim.git?rev=8efd1d5#8efd1d52490ee6e0b4619642d6908e15fdcc4d7c" dependencies = [ "aws-sdk-s3", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "aws-types", "bytes", "http", @@ -5425,7 +5480,7 @@ dependencies = [ "itertools 0.10.5", "log", "oauth2", - "p256", + "p256 0.13.2", "p384", "rand", "rsa", @@ -5665,14 +5720,25 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", + "sha2", +] + [[package]] name = "p256" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.8", + "elliptic-curve 0.13.6", "primeorder", "sha2", ] @@ -5683,8 +5749,8 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70786f51bcc69f6a4c0360e063a4cac5419ef7c5cd5b3c99ad70f3be5ba79209" dependencies = [ - "ecdsa", - "elliptic-curve", + "ecdsa 0.16.8", + "elliptic-curve 0.13.6", "primeorder", "sha2", ] @@ -6035,9 +6101,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.8", + "pkcs8 0.10.2", + "spki 0.7.2", +] + +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der 0.6.1", + "spki 0.6.0", ] [[package]] @@ -6046,8 +6122,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.8", + "spki 0.7.2", ] [[package]] @@ -6292,7 +6368,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c2fcef82c0ec6eefcc179b978446c399b3cdf73c392c35604e399eee6df1ee3" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.6", ] [[package]] @@ -6981,7 +7057,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -6991,7 +7067,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.7", + "rustls", "rustls-native-certs", "rustls-pemfile", "serde", @@ -6999,7 +7075,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -7011,6 +7087,17 @@ dependencies = [ "winreg", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -7040,11 +7127,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "risedev" version = "1.3.0-alpha" @@ -7160,6 +7261,7 @@ dependencies = [ "aws-config", "aws-sdk-s3", "aws-smithy-http", + "aws-smithy-types", "bcc", "bytes", "bytesize", @@ -7485,6 +7587,8 @@ dependencies = [ "aws-sdk-kinesis", "aws-sdk-s3", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "aws-types", "base64 0.21.4", "byteorder", @@ -8016,8 +8120,9 @@ dependencies = [ "async-trait", "await-tree", "aws-config", - "aws-smithy-client", "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "crc32fast", @@ -8025,6 +8130,7 @@ dependencies = [ "fail", "futures", "hyper", + "hyper-rustls", "hyper-tls", "itertools 0.11.0", "madsim-aws-sdk-s3", @@ -8032,6 +8138,7 @@ dependencies = [ "opendal", "prometheus", "risingwave_common", + "rustls", "spin 0.9.8", "thiserror", "tokio-retry", @@ -8503,10 +8610,10 @@ dependencies = [ "num-iter", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core", - "signature", - "spki", + "signature 2.0.0", + "spki 0.7.2", "subtle", "zeroize", ] @@ -8602,25 +8709,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.9" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" dependencies = [ "log", - "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.4", + "ring 0.17.5", + "rustls-webpki 0.101.7", "sct", ] @@ -8651,18 +8746,18 @@ version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -8750,8 +8845,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -8931,16 +9026,30 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct 0.1.1", + "der 0.6.1", + "generic-array", + "pkcs8 0.9.0", + "subtle", + "zeroize", +] + [[package]] name = "sec1" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.8", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -9284,12 +9393,22 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "rand_core", - "signature", + "signature 2.0.0", "zeroize", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "signature" version = "2.0.0" @@ -9461,6 +9580,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der 0.6.1", +] + [[package]] name = "spki" version = "0.7.2" @@ -9468,7 +9597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" dependencies = [ "base64ct", - "der", + "der 0.7.8", ] [[package]] @@ -10254,24 +10383,13 @@ dependencies = [ "rand", ] -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.9", - "tokio", - "webpki", -] - [[package]] name = "tokio-rustls" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.7", + "rustls", "tokio", ] @@ -10391,7 +10509,7 @@ dependencies = [ "prost 0.11.9", "rustls-pemfile", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", @@ -10741,6 +10859,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.1" @@ -10964,16 +11088,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki-roots" version = "0.23.1" @@ -11227,8 +11341,10 @@ dependencies = [ "async-std", "auto_enums", "aws-credential-types", + "aws-runtime", "aws-sdk-s3", - "aws-smithy-client", + "aws-sigv4", + "aws-smithy-runtime", "base64 0.21.4", "bit-vec", "bitflags 2.4.0", @@ -11306,9 +11422,9 @@ dependencies = [ "regex-automata 0.4.1", "regex-syntax 0.8.0", "reqwest", - "ring", + "ring 0.16.20", "rust_decimal", - "rustls 0.21.7", + "rustls", "scopeguard", "sea-orm", "sea-query", @@ -11417,11 +11533,11 @@ dependencies = [ "futures", "http", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls", "itertools 0.10.5", "log", "percent-encoding", - "rustls 0.21.7", + "rustls", "rustls-pemfile", "seahash", "serde", @@ -11506,3 +11622,8 @@ dependencies = [ "libc", "pkg-config", ] + +[[patch.unused]] +name = "getrandom" +version = "0.2.9" +source = "git+https://github.com/madsim-rs/getrandom.git?rev=8daf97e#8daf97e4142635fe28543b2db9022f5e2544bb5c" diff --git a/Cargo.toml b/Cargo.toml index 46d3a60487b89..110940e912e4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,37 +67,30 @@ repository = "https://github.com/risingwavelabs/risingwave" [workspace.dependencies] await-tree = "0.1.1" -aws-config = { version = "0.55", default-features = false, features = [ +aws-config = { version = "0.57", default-features = false, features = [ "rt-tokio", - "native-tls", ] } -aws-credential-types = { version = "0.55", default-features = false, features = [ +aws-credential-types = { version = "0.57", default-features = false, features = [ "hardcoded-credentials", ] } -aws-sdk-kinesis = { version = "0.28", default-features = false, features = [ +aws-sdk-kinesis = { version = "0.35", default-features = false, features = [ "rt-tokio", - "native-tls", ] } -aws-sdk-s3 = { version = "0.28", default-features = false, features = [ +aws-sdk-s3 = { version = "0.35", default-features = false, features = [ "rt-tokio", - "native-tls", ] } -aws-sdk-ec2 = { version = "0.28", default-features = false, features = [ +aws-sdk-ec2 = { version = "0.35", default-features = false, features = [ "rt-tokio", - "native-tls", ] } -aws-sdk-sqs = { version = "0.28", default-features = false, features = [ +aws-sdk-sqs = { version = "0.35", default-features = false, features = [ "rt-tokio", - "native-tls", ] } -aws-smithy-client = { version = "0.55", default-features = false, features = [ - "rt-tokio", - "native-tls", -] } -aws-smithy-http = "0.55" -aws-smithy-types = "0.55" -aws-endpoint = "0.55" -aws-types = "0.55" +aws-smithy-http = "0.57" +aws-smithy-runtime-api = "0.57" +aws-smithy-runtime = "0.57" +aws-smithy-types = "0.57" +aws-endpoint = "0.57" +aws-types = "0.57" etcd-client = { package = "madsim-etcd-client", version = "0.4" } futures-async-stream = "0.2.9" hytra = "0.1" diff --git a/src/bench/Cargo.toml b/src/bench/Cargo.toml index 31b8f5ce7cc94..5227c39272c38 100644 --- a/src/bench/Cargo.toml +++ b/src/bench/Cargo.toml @@ -12,6 +12,7 @@ async-trait = "0.1" aws-config = { workspace = true } aws-sdk-s3 = { workspace = true } aws-smithy-http = { workspace = true } +aws-smithy-types = { workspace = true } bcc = { version = "0.0.33", optional = true } bytes = "1" bytesize = { version = "1", features = ["serde"] } diff --git a/src/bench/s3_bench/main.rs b/src/bench/s3_bench/main.rs index f6cbb7cf89200..f0b5d99bd9ff2 100644 --- a/src/bench/s3_bench/main.rs +++ b/src/bench/s3_bench/main.rs @@ -21,7 +21,7 @@ use std::time::{Duration, Instant}; use aws_config::retry::RetryConfig; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use aws_sdk_s3::Client; -use aws_smithy_http::body::SdkBody; +use aws_smithy_types::body::SdkBody; use bytesize::ByteSize; use clap::Parser; use futures::stream::{self, StreamExt}; diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index 3bec35b5030df..cc5deac51c72b 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -32,6 +32,8 @@ aws-credential-types = { workspace = true } aws-sdk-kinesis = { workspace = true } aws-sdk-s3 = { workspace = true } aws-smithy-http = { workspace = true } +aws-smithy-runtime-api = { workspace = true } +aws-smithy-types = { workspace = true } aws-types = { workspace = true } base64 = "0.21" byteorder = "1" diff --git a/src/connector/src/aws_auth.rs b/src/connector/src/aws_auth.rs index d5c6e3fc6b971..cf31744c82e45 100644 --- a/src/connector/src/aws_auth.rs +++ b/src/connector/src/aws_auth.rs @@ -114,7 +114,8 @@ impl AwsAuthProps { if let Some(id) = &self.external_id { role = role.external_id(id); } - Ok(SharedCredentialsProvider::new(role.build(credential))) + let provider = role.build_from_provider(credential).await; + Ok(SharedCredentialsProvider::new(provider)) } else { Ok(credential) } diff --git a/src/connector/src/source/filesystem/s3/source/reader.rs b/src/connector/src/source/filesystem/s3/source/reader.rs index b1e368a2b409e..8a9052c6e4511 100644 --- a/src/connector/src/source/filesystem/s3/source/reader.rs +++ b/src/connector/src/source/filesystem/s3/source/reader.rs @@ -19,10 +19,12 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use aws_sdk_s3::client as s3_client; use aws_sdk_s3::operation::get_object::GetObjectError; -use aws_smithy_http::byte_stream::ByteStream; -use aws_smithy_http::result::SdkError; -use futures::TryStreamExt; +use aws_smithy_http::futures_stream_adapter::FuturesStreamCompatByteStream; +use aws_smithy_runtime_api::client::result::SdkError; +use aws_smithy_types::body::SdkBody; +use aws_smithy_types::byte_stream::ByteStream; use futures_async_stream::try_stream; +use hyper::Response; use io::StreamReader; use risingwave_common::error::RwError; use tokio::io::BufReader; @@ -91,9 +93,10 @@ impl S3FileReader { } }; - let stream_reader = StreamReader::new( - byte_stream.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)), - ); + // FYI: https://github.com/awslabs/smithy-rs/pull/2983 + let byte_stream = FuturesStreamCompatByteStream::new(byte_stream); + + let stream_reader = StreamReader::new(byte_stream); let reader = pin!(BufReader::new(stream_reader)); @@ -142,7 +145,7 @@ impl S3FileReader { bucket_name: &str, object_name: &str, start: usize, - ) -> std::result::Result> { + ) -> std::result::Result>> { let range = if start == 0 { None } else { diff --git a/src/connector/src/source/kinesis/enumerator/client.rs b/src/connector/src/source/kinesis/enumerator/client.rs index 59d1aca06e4f1..8394fae6e8f43 100644 --- a/src/connector/src/source/kinesis/enumerator/client.rs +++ b/src/connector/src/source/kinesis/enumerator/client.rs @@ -75,7 +75,7 @@ impl SplitEnumerator for KinesisSplitEnumerator { Ok(shard_collect .into_iter() .map(|x| KinesisSplit { - shard_id: x.shard_id().unwrap_or_default().to_string().into(), + shard_id: x.shard_id().to_string().into(), // handle start with position in reader part start_position: KinesisOffset::None, end_position: KinesisOffset::None, diff --git a/src/connector/src/source/kinesis/source/message.rs b/src/connector/src/source/kinesis/source/message.rs index a4daa7f1195bd..9e63c05be0e0f 100644 --- a/src/connector/src/source/kinesis/source/message.rs +++ b/src/connector/src/source/kinesis/source/message.rs @@ -40,9 +40,9 @@ impl KinesisMessage { pub fn new(shard_id: SplitId, message: Record) -> Self { KinesisMessage { shard_id, - sequence_number: message.sequence_number.unwrap(), - partition_key: message.partition_key.unwrap(), - payload: message.data.unwrap().into_inner(), + sequence_number: message.sequence_number, + partition_key: message.partition_key, + payload: message.data.into_inner(), } } } diff --git a/src/connector/src/source/kinesis/source/reader.rs b/src/connector/src/source/kinesis/source/reader.rs index 4ed00b88a58b2..8ba465342c2ce 100644 --- a/src/connector/src/source/kinesis/source/reader.rs +++ b/src/connector/src/source/kinesis/source/reader.rs @@ -137,7 +137,7 @@ impl CommonSplitReader for KinesisSplitReader { match self.get_records().await { Ok(resp) => { self.shard_iter = resp.next_shard_iterator().map(String::from); - let chunk = (resp.records().unwrap().iter()) + let chunk = (resp.records().iter()) .map(|r| { SourceMessage::from(KinesisMessage::new( self.shard_id.clone(), diff --git a/src/meta/src/rpc/cloud_provider.rs b/src/meta/src/rpc/cloud_provider.rs index 2b307f955e873..bcf8369759b40 100644 --- a/src/meta/src/rpc/cloud_provider.rs +++ b/src/meta/src/rpc/cloud_provider.rs @@ -62,14 +62,12 @@ impl AwsEc2Client { ) })?; - if let Some(ret) = output.unsuccessful() { - if !ret.is_empty() { - return Err(MetaError::from(anyhow!( - "Failed to delete VPC endpoint {}, error: {:?}", - vpc_endpoint_id, - ret - ))); - } + if !output.unsuccessful().is_empty() { + return Err(MetaError::from(anyhow!( + "Failed to delete VPC endpoint {}, error: {:?}", + vpc_endpoint_id, + output.unsuccessful() + ))); } Ok(()) } @@ -325,13 +323,11 @@ impl AwsEc2Client { let endpoint = output.vpc_endpoint().unwrap(); let mut dns_names = Vec::new(); - if let Some(dns_entries) = endpoint.dns_entries() { - dns_entries.iter().for_each(|e| { - if let Some(dns_name) = e.dns_name() { - dns_names.push(dns_name.to_string()); - } - }); - } + endpoint.dns_entries().iter().for_each(|e| { + if let Some(dns_name) = e.dns_name() { + dns_names.push(dns_name.to_string()); + } + }); Ok(( endpoint.vpc_endpoint_id().unwrap_or_default().to_string(), diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index 2459bf83b5af6..96a4d610c15be 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -12,9 +12,10 @@ repository = { workspace = true } async-trait = "0.1" await-tree = { workspace = true } aws-config = { workspace = true } -aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3" } -aws-smithy-client = { workspace = true } +aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3", git = "https://github.com/mrcroxx/madsim.git", rev = "8efd1d5" } aws-smithy-http = { workspace = true } +aws-smithy-runtime = { workspace = true } +aws-smithy-runtime-api = { workspace = true } aws-smithy-types = { workspace = true } bytes = { version = "1", features = ["serde"] } crc32fast = "1.3.2" @@ -22,11 +23,13 @@ either = "1" fail = "0.5" futures = { version = "0.3", default-features = false, features = ["alloc"] } hyper = "0.14" +hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] } hyper-tls = "0.5.0" itertools = "0.11" opendal = "0.41" prometheus = { version = "0.13", features = ["process"] } risingwave_common = { workspace = true } +rustls = "0.21.8" spin = "0.9" thiserror = "1" tokio = { version = "0.2", package = "madsim-tokio", features = ["fs"] } diff --git a/src/object_store/src/object/error.rs b/src/object_store/src/object/error.rs index 00877688347ff..7a432a45fdc57 100644 --- a/src/object_store/src/object/error.rs +++ b/src/object_store/src/object/error.rs @@ -20,6 +20,8 @@ use aws_sdk_s3::error::DisplayErrorContext; use aws_sdk_s3::operation::get_object::GetObjectError; use aws_sdk_s3::operation::head_object::HeadObjectError; use aws_sdk_s3::primitives::ByteStreamError; +use aws_smithy_types::body::SdkBody; +use hyper::Response; use risingwave_common::error::BoxedError; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; @@ -79,13 +81,19 @@ impl ObjectError { pub fn is_object_not_found_error(&self) -> bool { match &self.inner { ObjectErrorInner::S3(e) => { - if let Some(aws_smithy_http::result::SdkError::ServiceError(err)) = - e.downcast_ref::>() + if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) = e + .downcast_ref::, + >>() { return matches!(err.err(), GetObjectError::NoSuchKey(_)); } - if let Some(aws_smithy_http::result::SdkError::ServiceError(err)) = - e.downcast_ref::>() + if let Some(aws_smithy_runtime_api::client::result::SdkError::ServiceError(err)) = e + .downcast_ref::, + >>() { return matches!(err.err(), HeadObjectError::NotFound(_)); } diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs index 6dce56ed4c759..613b36b176080 100644 --- a/src/object_store/src/object/s3.rs +++ b/src/object_store/src/object/s3.rs @@ -30,15 +30,16 @@ use aws_sdk_s3::types::{ CompletedPart, Delete, ExpirationStatus, LifecycleRule, LifecycleRuleFilter, ObjectIdentifier, }; use aws_sdk_s3::Client; -use aws_smithy_client::http_connector::{ConnectorSettings, HttpConnector}; -use aws_smithy_http::body::SdkBody; -use aws_smithy_http::result::SdkError; +use aws_smithy_runtime::client::http::hyper_014::HyperClientBuilder; +use aws_smithy_runtime_api::client::http::HttpClient; +use aws_smithy_runtime_api::client::result::SdkError; +use aws_smithy_types::body::SdkBody; use aws_smithy_types::retry::RetryConfig; use either::Either; use fail::fail_point; use futures::future::{try_join_all, BoxFuture, FutureExt}; use futures::{stream, Stream}; -use hyper::Body; +use hyper::{Body, Response}; use itertools::Itertools; use risingwave_common::config::default::s3_objstore_config; use risingwave_common::monitor::connection::monitor_connector; @@ -293,7 +294,9 @@ impl StreamingUploader for S3StreamingUploader { fn get_upload_body(data: Vec) -> ByteStream { SdkBody::retryable(move || { - Body::wrap_stream(stream::iter(data.clone().into_iter().map(ObjectResult::Ok))).into() + SdkBody::from_body_0_4(Body::wrap_stream(stream::iter( + data.clone().into_iter().map(ObjectResult::Ok), + ))) }) .into() } @@ -495,7 +498,7 @@ impl ObjectStore for S3ObjectStore { // Create identifiers from paths. let mut obj_ids = Vec::with_capacity(slice.len()); for path in slice { - obj_ids.push(ObjectIdentifier::builder().key(path).build()); + obj_ids.push(ObjectIdentifier::builder().key(path).build().unwrap()); } // Build and submit request to delete objects. @@ -504,12 +507,12 @@ impl ObjectStore for S3ObjectStore { .client .delete_objects() .bucket(&self.bucket) - .delete(delete_builder.build()).send() + .delete(delete_builder.build().unwrap()).send() .await?; // Check if there were errors. - if let Some(err_list) = delete_output.errors() && !err_list.is_empty() { - return Err(ObjectError::internal(format!("DeleteObjects request returned exception for some objects: {:?}", err_list))); + if !delete_output.errors().is_empty() { + return Err(ObjectError::internal(format!("DeleteObjects request returned exception for some objects: {:?}", delete_output.errors()))); } } @@ -537,41 +540,37 @@ impl S3ObjectStore { Self::new_with_config(bucket, metrics, S3ObjectStoreConfig::default()).await } - pub fn new_http_connector(config: &S3ObjectStoreConfig) -> impl Into { - // Customize http connector to set keepalive. - let native_tls = { - let mut tls = hyper_tls::native_tls::TlsConnector::builder(); - let tls = tls - .min_protocol_version(Some(hyper_tls::native_tls::Protocol::Tlsv12)) - .build() - .unwrap_or_else(|e| panic!("Error while creating TLS connector: {}", e)); - let mut http = hyper::client::HttpConnector::new(); + pub fn new_http_client(config: &S3ObjectStoreConfig) -> impl HttpClient { + let mut http = hyper::client::HttpConnector::new(); - // connection config - if let Some(keepalive_ms) = config.keepalive_ms.as_ref() { - http.set_keepalive(Some(Duration::from_millis(*keepalive_ms))); - } + // connection config + if let Some(keepalive_ms) = config.keepalive_ms.as_ref() { + http.set_keepalive(Some(Duration::from_millis(*keepalive_ms))); + } - if let Some(nodelay) = config.nodelay.as_ref() { - http.set_nodelay(*nodelay); - } + if let Some(nodelay) = config.nodelay.as_ref() { + http.set_nodelay(*nodelay); + } - if let Some(recv_buffer_size) = config.recv_buffer_size.as_ref() { - http.set_recv_buffer_size(Some(*recv_buffer_size)); - } + if let Some(recv_buffer_size) = config.recv_buffer_size.as_ref() { + http.set_recv_buffer_size(Some(*recv_buffer_size)); + } - if let Some(send_buffer_size) = config.send_buffer_size.as_ref() { - http.set_send_buffer_size(Some(*send_buffer_size)); - } + if let Some(send_buffer_size) = config.send_buffer_size.as_ref() { + http.set_send_buffer_size(Some(*send_buffer_size)); + } - http.enforce_http(false); - hyper_tls::HttpsConnector::from((http, tls.into())) - }; + http.enforce_http(false); + + let conn = hyper_rustls::HttpsConnectorBuilder::new() + .with_webpki_roots() + .https_or_http() + .enable_all_versions() + .wrap_connector(http); - aws_smithy_client::hyper_ext::Adapter::builder() - .hyper_builder(hyper::client::Builder::default()) - .connector_settings(ConnectorSettings::builder().build()) - .build(monitor_connector(native_tls, "S3")) + let conn = monitor_connector(conn, "S3"); + + HyperClientBuilder::new().build(conn) } pub async fn new_with_config( @@ -581,7 +580,7 @@ impl S3ObjectStore { ) -> Self { let sdk_config_loader = aws_config::from_env() .retry_config(RetryConfig::standard().with_max_attempts(4)) - .http_connector(Self::new_http_connector(&config)); + .http_client(Self::new_http_client(&config)); // Retry 3 times if we get server-side errors or throttling errors let client = match std::env::var("RW_S3_ENDPOINT") { @@ -643,7 +642,7 @@ impl S3ObjectStore { let builder = aws_sdk_s3::config::Builder::from(&aws_config::ConfigLoader::default().load().await) .force_path_style(true) - .http_connector(Self::new_http_connector(&S3ObjectStoreConfig::default())); + .http_client(Self::new_http_client(&S3ObjectStoreConfig::default())); let config = builder .region(Region::new("custom")) .endpoint_url(format!("{}{}", endpoint_prefix, address)) @@ -654,6 +653,7 @@ impl S3ObjectStore { )) .build(); let client = Client::from_conf(config); + Self { client, bucket: bucket.to_string(), @@ -717,11 +717,11 @@ impl S3ObjectStore { .await; let mut is_expiration_configured = false; if let Ok(config) = &get_config_result { - for rule in config.rules().unwrap_or_default() { + for rule in config.rules() { if rule.expiration().is_some() { is_expiration_configured = true; } - if matches!(rule.status().unwrap(), ExpirationStatus::Enabled) + if matches!(rule.status(), ExpirationStatus::Enabled) && rule.abort_incomplete_multipart_upload().is_some() { configured_rules.push(rule); @@ -745,10 +745,12 @@ impl S3ObjectStore { .days_after_initiation(S3_INCOMPLETE_MULTIPART_UPLOAD_RETENTION_DAYS) .build(), ) - .build(); + .build() + .unwrap(); let bucket_lifecycle_config = BucketLifecycleConfiguration::builder() .rules(bucket_lifecycle_rule) - .build(); + .build() + .unwrap(); if self .client .put_bucket_lifecycle_configuration() @@ -777,7 +779,9 @@ impl S3ObjectStore { } #[inline(always)] - fn should_retry(err: &Either, ByteStreamError>) -> bool { + fn should_retry( + err: &Either>, ByteStreamError>, + ) -> bool { match err { Either::Left(err) => { if let SdkError::DispatchFailure(e) = err { @@ -853,7 +857,10 @@ struct S3ObjectIter { send_future: Option< BoxFuture< 'static, - Result<(Vec, Option, bool), SdkError>, + Result< + (Vec, Option, bool), + SdkError>, + >, >, >, } @@ -910,7 +917,6 @@ impl Stream for S3ObjectIter { Ok(r) => { let more = r .contents() - .unwrap_or_default() .iter() .map(|obj| ObjectMetadata { key: obj.key().expect("key required").to_owned(), @@ -933,8 +939,8 @@ impl Stream for S3ObjectIter { } } -impl From, ByteStreamError>> for ObjectError { - fn from(e: Either, ByteStreamError>) -> Self { +impl From>, ByteStreamError>> for ObjectError { + fn from(e: Either>, ByteStreamError>) -> Self { match e { Either::Left(e) => e.into(), Either::Right(e) => e.into(), diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 734eccadd4c16..fb8b08a9f54d4 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -13,7 +13,7 @@ normal = ["serde"] [dependencies] anyhow = "1.0" async-trait = "0.1" -aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3" } +aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3", git = "https://github.com/mrcroxx/madsim.git", rev = "8efd1d5" } cfg-or-panic = "0.2" clap = { version = "4", features = ["derive"] } console = "0.15" diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index 0edc6c7bb5ec6..ffdca9ba8898d 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -22,9 +22,11 @@ ahash = { version = "0.8" } allocator-api2 = { version = "0.2", default-features = false, features = ["alloc", "nightly"] } anyhow = { version = "1", features = ["backtrace"] } async-std = { version = "1", features = ["attributes", "tokio1"] } -aws-credential-types = { version = "0.55", default-features = false, features = ["hardcoded-credentials"] } -aws-sdk-s3 = { version = "0.28", features = ["native-tls"] } -aws-smithy-client = { version = "0.55", default-features = false, features = ["native-tls", "rustls"] } +aws-credential-types = { version = "0.57", default-features = false, features = ["hardcoded-credentials"] } +aws-runtime = { version = "0.57", default-features = false, features = ["event-stream", "sigv4a"] } +aws-sdk-s3 = { version = "0.35" } +aws-sigv4 = { version = "0.57", features = ["http0-compat", "sign-eventstream", "sigv4a"] } +aws-smithy-runtime = { version = "0.57", default-features = false, features = ["client", "rt-tokio", "tls-rustls"] } base64 = { version = "0.21", features = ["alloc"] } bit-vec = { version = "0.6" } bitflags = { version = "2", default-features = false, features = ["serde", "std"] } @@ -132,7 +134,7 @@ toml_datetime = { version = "0.6", default-features = false, features = ["serde" toml_edit = { version = "0.19", features = ["serde"] } tonic-274715c4dabd11b0 = { package = "tonic", version = "0.9", features = ["gzip", "tls-webpki-roots"] } tonic-93f6ce9d446188ac = { package = "tonic", version = "0.10" } -tower = { version = "0.4", features = ["balance", "buffer", "filter", "limit", "load-shed", "retry", "timeout", "util"] } +tower = { version = "0.4", features = ["balance", "buffer", "filter", "limit", "load-shed", "timeout", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "local-time", "parking_lot"] } From 4a8ba891e6c3420dde3e3dcfaf08edf0ea89372b Mon Sep 17 00:00:00 2001 From: MrCroxx Date: Fri, 10 Nov 2023 15:19:21 +0800 Subject: [PATCH 2/3] chore: enable rustls feature for aws sdk, change madsim to v3.0 Signed-off-by: MrCroxx --- Cargo.lock | 5 +++-- Cargo.toml | 10 +++++----- src/object_store/Cargo.toml | 2 +- src/tests/simulation/Cargo.toml | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed03deac0f3fa..41cf9117ca4c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4695,8 +4695,9 @@ dependencies = [ [[package]] name = "madsim-aws-sdk-s3" -version = "0.2.25+0.28" -source = "git+https://github.com/mrcroxx/madsim.git?rev=8efd1d5#8efd1d52490ee6e0b4619642d6908e15fdcc4d7c" +version = "0.3.0+0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f484435d52fc8f9af55a687b7777fcdd8fd9acc01901526367137fc39473c56" dependencies = [ "aws-sdk-s3", "aws-smithy-http", diff --git a/Cargo.toml b/Cargo.toml index 110940e912e4f..c730b7ede8d00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,22 +68,22 @@ repository = "https://github.com/risingwavelabs/risingwave" [workspace.dependencies] await-tree = "0.1.1" aws-config = { version = "0.57", default-features = false, features = [ - "rt-tokio", + "rt-tokio", "rustls", ] } aws-credential-types = { version = "0.57", default-features = false, features = [ "hardcoded-credentials", ] } aws-sdk-kinesis = { version = "0.35", default-features = false, features = [ - "rt-tokio", + "rt-tokio", "rustls", ] } aws-sdk-s3 = { version = "0.35", default-features = false, features = [ - "rt-tokio", + "rt-tokio", "rustls", ] } aws-sdk-ec2 = { version = "0.35", default-features = false, features = [ - "rt-tokio", + "rt-tokio", "rustls", ] } aws-sdk-sqs = { version = "0.35", default-features = false, features = [ - "rt-tokio", + "rt-tokio", "rustls", ] } aws-smithy-http = "0.57" aws-smithy-runtime-api = "0.57" diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index 96a4d610c15be..87fd07be7810a 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -12,7 +12,7 @@ repository = { workspace = true } async-trait = "0.1" await-tree = { workspace = true } aws-config = { workspace = true } -aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3", git = "https://github.com/mrcroxx/madsim.git", rev = "8efd1d5" } +aws-sdk-s3 = { version = "0.3", package = "madsim-aws-sdk-s3" } aws-smithy-http = { workspace = true } aws-smithy-runtime = { workspace = true } aws-smithy-runtime-api = { workspace = true } diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index fb8b08a9f54d4..55a2f5c69684b 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -13,7 +13,7 @@ normal = ["serde"] [dependencies] anyhow = "1.0" async-trait = "0.1" -aws-sdk-s3 = { version = "0.2", package = "madsim-aws-sdk-s3", git = "https://github.com/mrcroxx/madsim.git", rev = "8efd1d5" } +aws-sdk-s3 = { version = "0.3", package = "madsim-aws-sdk-s3" } cfg-or-panic = "0.2" clap = { version = "4", features = ["derive"] } console = "0.15" From ace24effacc9eac325aa672e2670cb680df58670 Mon Sep 17 00:00:00 2001 From: MrCroxx Date: Tue, 14 Nov 2023 12:33:44 +0800 Subject: [PATCH 3/3] fix: use hyper-0-14-x feature instead of http-body-0-4 Signed-off-by: MrCroxx --- Cargo.lock | 2 ++ Cargo.toml | 21 ++++++++++++++------- src/object_store/src/object/s3.rs | 4 +--- src/workspace-hack/Cargo.toml | 1 + 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fab12d3cdbb20..c2571511fd9b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,6 +1020,7 @@ dependencies = [ "futures-core", "http", "http-body", + "hyper", "itoa", "num-integer", "pin-project-lite", @@ -11398,6 +11399,7 @@ dependencies = [ "aws-sdk-s3", "aws-sigv4", "aws-smithy-runtime", + "aws-smithy-types", "base64 0.21.4", "bit-vec", "bitflags 2.4.0", diff --git a/Cargo.toml b/Cargo.toml index a204aaf76cad8..7820fa06cba81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,27 +69,32 @@ repository = "https://github.com/risingwavelabs/risingwave" [workspace.dependencies] await-tree = "0.1.1" aws-config = { version = "0.57", default-features = false, features = [ - "rt-tokio", "rustls", + "rt-tokio", + "rustls", ] } aws-credential-types = { version = "0.57", default-features = false, features = [ "hardcoded-credentials", ] } aws-sdk-kinesis = { version = "0.35", default-features = false, features = [ - "rt-tokio", "rustls", + "rt-tokio", + "rustls", ] } aws-sdk-s3 = { version = "0.35", default-features = false, features = [ - "rt-tokio", "rustls", + "rt-tokio", + "rustls", ] } aws-sdk-ec2 = { version = "0.35", default-features = false, features = [ - "rt-tokio", "rustls", + "rt-tokio", + "rustls", ] } aws-sdk-sqs = { version = "0.35", default-features = false, features = [ - "rt-tokio", "rustls", + "rt-tokio", + "rustls", ] } aws-smithy-http = "0.57" aws-smithy-runtime-api = "0.57" aws-smithy-runtime = "0.57" -aws-smithy-types = "0.57" +aws-smithy-types = { version = "0.57", default-features = false, features = ["hyper-0-14-x"] } aws-endpoint = "0.57" aws-types = "0.57" etcd-client = { package = "madsim-etcd-client", version = "0.4" } @@ -107,7 +112,9 @@ criterion = { version = "0.5", features = ["async_futures"] } tonic = { package = "madsim-tonic", version = "0.4.0" } tonic-build = { package = "madsim-tonic-build", version = "0.4.2" } prost = { version = "0.12" } -icelake = { git = "https://github.com/icelake-io/icelake", rev = "5cdcdffd24f4624a0a43f92c5f368988169a799b", features = ["prometheus"] } +icelake = { git = "https://github.com/icelake-io/icelake", rev = "5cdcdffd24f4624a0a43f92c5f368988169a799b", features = [ + "prometheus", +] } arrow-array = "48" arrow-cast = "48" arrow-schema = "48" diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs index 7e59669b6a920..bc826cbbd2ec8 100644 --- a/src/object_store/src/object/s3.rs +++ b/src/object_store/src/object/s3.rs @@ -294,9 +294,7 @@ impl StreamingUploader for S3StreamingUploader { fn get_upload_body(data: Vec) -> ByteStream { SdkBody::retryable(move || { - SdkBody::from_body_0_4(Body::wrap_stream(stream::iter( - data.clone().into_iter().map(ObjectResult::Ok), - ))) + Body::wrap_stream(stream::iter(data.clone().into_iter().map(ObjectResult::Ok))).into() }) .into() } diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index 6707f136a934e..b263a9c5bcefb 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -27,6 +27,7 @@ aws-runtime = { version = "0.57", default-features = false, features = ["event-s aws-sdk-s3 = { version = "0.35" } aws-sigv4 = { version = "0.57", features = ["http0-compat", "sign-eventstream", "sigv4a"] } aws-smithy-runtime = { version = "0.57", default-features = false, features = ["client", "rt-tokio", "tls-rustls"] } +aws-smithy-types = { version = "0.57", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "hyper-0-14-x", "rt-tokio"] } base64 = { version = "0.21", features = ["alloc"] } bit-vec = { version = "0.6" } bitflags = { version = "2", default-features = false, features = ["serde", "std"] }