chore(object store): change streaming write buffer size to 16MB for gcs (risingwavelabs#12150)

Author: wcy-fdu

src/object_store/src/object/opendal_engine/gcs.rs

@@ -18,6 +18,13 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+/// The fixed number of bytes that is buffered before they are uploaded as a part, will be used in
+/// streaing upload.
+///
+/// Reference: <https://cloud.google.com/storage/docs/streaming-uploads>
+const GCS_PART_SIZE: usize = 16 * 1024 * 1024;
+
 impl OpendalObjectStore {
     /// create opendal gcs engine.
     pub fn new_gcs_engine(bucket: String, root: String) -> ObjectResult<Self> {
@@ -28,6 +35,8 @@ impl OpendalObjectStore {
 
         builder.root(&root);
 
+        builder.write_fixed_size(GCS_PART_SIZE);
+
         // if credential env is set, use it. Otherwise, ADC will be used.
         let cred = std::env::var("GOOGLE_APPLICATION_CREDENTIALS");
         if let Ok(cred) = cred {

src/object_store/src/object/opendal_engine/oss.rs

@@ -18,6 +18,13 @@ use opendal::Operator;
 
 use super::{EngineType, OpendalObjectStore};
 use crate::object::ObjectResult;
+
+/// The minimum number of bytes that is buffered before they are uploaded as a part, , will be used
+/// in streaing upload.
+///
+/// Reference: <https://www.alibabacloud.com/help/en/oss/user-guide/multipart-upload-12>
+const OSS_PART_SIZE: usize = 16 * 1024 * 1024;
+
 impl OpendalObjectStore {
     /// create opendal oss engine.
     pub fn new_oss_engine(bucket: String, root: String) -> ObjectResult<Self> {
@@ -26,6 +33,8 @@ impl OpendalObjectStore {
 
         builder.bucket(&bucket);
 
+        builder.write_min_size(OSS_PART_SIZE);
+
         builder.root(&root);
 
         let endpoint = std::env::var("OSS_ENDPOINT")