From c125ec4c15150a51b9edaa0c36790c9b2708d1f7 Mon Sep 17 00:00:00 2001
From: congyi wang <58715567+wcy-fdu@users.noreply.github.com>
Date: Fri, 8 Sep 2023 12:16:06 +0800
Subject: [PATCH] chore(object store): change streaming write buffer size to
 16MB for gcs (#12150)

---
 src/object_store/src/object/opendal_engine/gcs.rs | 9 +++++++++
 src/object_store/src/object/opendal_engine/oss.rs | 9 +++++++++
 2 files changed, 18 insertions(+)

diff --git a/src/object_store/src/object/opendal_engine/gcs.rs b/src/object_store/src/object/opendal_engine/gcs.rs
index 206a5ab6b4e08..bb6ef8eee0446 100644
--- a/src/object_store/src/object/opendal_engine/gcs.rs
+++ b/src/object_store/src/object/opendal_engine/gcs.rs
@@ -18,6 +18,13 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+/// The fixed number of bytes that is buffered before they are uploaded as a part, will be used in
+/// streaing upload.
+///
+/// Reference: <https://cloud.google.com/storage/docs/streaming-uploads>
+const GCS_PART_SIZE: usize = 16 * 1024 * 1024;
+
 impl OpendalObjectStore {
     /// create opendal gcs engine.
     pub fn new_gcs_engine(bucket: String, root: String) -> ObjectResult<Self> {
@@ -28,6 +35,8 @@ impl OpendalObjectStore {
 
         builder.root(&root);
 
+        builder.write_fixed_size(GCS_PART_SIZE);
+
         // if credential env is set, use it. Otherwise, ADC will be used.
         let cred = std::env::var("GOOGLE_APPLICATION_CREDENTIALS");
         if let Ok(cred) = cred {
diff --git a/src/object_store/src/object/opendal_engine/oss.rs b/src/object_store/src/object/opendal_engine/oss.rs
index 292ddf2c614f1..f10458402ec76 100644
--- a/src/object_store/src/object/opendal_engine/oss.rs
+++ b/src/object_store/src/object/opendal_engine/oss.rs
@@ -18,6 +18,13 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+/// The minimum number of bytes that is buffered before they are uploaded as a part, , will be used
+/// in streaing upload.
+///
+/// Reference: <https://www.alibabacloud.com/help/en/oss/user-guide/multipart-upload-12>
+const OSS_PART_SIZE: usize = 16 * 1024 * 1024;
+
 impl OpendalObjectStore {
     /// create opendal oss engine.
     pub fn new_oss_engine(bucket: String, root: String) -> ObjectResult<Self> {
@@ -26,6 +33,8 @@ impl OpendalObjectStore {
 
         builder.bucket(&bucket);
 
+        builder.write_min_size(OSS_PART_SIZE);
+
         builder.root(&root);
 
         let endpoint = std::env::var("OSS_ENDPOINT")