Refactor S3 store & support upload retry (TraceMachina#854)

* Upload failures for S3 will now retry
* S3 tcp connections now counted as with file descriptor limits
* Significantly refactored how uploads happen in S3 store

nativelink-config/src/stores.rs

@@ -480,6 +480,14 @@ pub struct S3Store {
     #[serde(default)]
     pub retry: Retry,
 
+    /// The maximum buffer size to retain in case of a retryable error
+    /// during upload. Setting this to zero will disable upload buffering;
+    /// this means that in the event of a failure during upload, the entire
+    /// upload will be aborted and the client will likely receive an error.
+    ///
+    /// Default: 5MB.
+    pub max_retry_buffer_per_request: Option<usize>,
+
     /// Maximum number of concurrent UploadPart requests per MultipartUpload.
     ///
     /// Default: 10.

nativelink-store/BUILD.bazel

@@ -49,8 +49,10 @@ rust_library(
         "@crates//:filetime",
         "@crates//:futures",
         "@crates//:hex",
+        "@crates//:http-body",
         "@crates//:hyper",
         "@crates//:hyper-rustls",
+        "@crates//:log",
         "@crates//:lz4_flex",
         "@crates//:parking_lot",
         "@crates//:prost",
@@ -98,12 +100,14 @@ rust_test_suite(
         "@crates//:async-lock",
         "@crates//:aws-sdk-s3",
         "@crates//:aws-smithy-runtime",
+        "@crates//:aws-smithy-runtime-api",
         "@crates//:aws-smithy-types",
         "@crates//:bincode",
         "@crates//:bytes",
         "@crates//:filetime",
         "@crates//:futures",
         "@crates//:http",
+        "@crates//:http-body",
         "@crates//:hyper",
         "@crates//:memory-stats",
         "@crates//:once_cell",

nativelink-store/Cargo.toml

@@ -21,8 +21,10 @@ bytes = "1.6.0"
 filetime = "0.2.23"
 futures = "0.3.30"
 hex = "0.4.3"
+http-body = "1.0.0"
 hyper = { version = "0.14.28" }
 hyper-rustls = { version = "0.24.2", features = ["webpki-tokio"] }
+log = "0.4.21"
 lz4_flex = "0.11.3"
 parking_lot = "0.12.1"
 prost = "0.12.4"
@@ -46,3 +48,4 @@ http = "1.1.0"
 aws-smithy-types = "1.1.8"
 aws-sdk-s3 = { version = "1.21.0"  }
 aws-smithy-runtime = { version = "1.2.1", features = ["test-util"] }
+aws-smithy-runtime-api = "1.4.0"