Merge pull request #210 from ywk253100/231010_buffer

Adjust the buffer size to avoid OOM

Author: shubham-pampattiwar

backupstoragelocation.md

@@ -70,6 +70,6 @@ spec:
     # See https://docs.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs#about-block-blobs
     # for more information on block blobs.
     #
-    # Optional (defaults to 104857600, i.e. 100MB).
-    blockSizeInBytes: "104857600"
+    # Optional (defaults to 1048576, i.e. 1MB, maximum 104857600, i.e. 100MB).
+    blockSizeInBytes: "1048576"
 ```

velero-plugin-for-microsoft-azure/object_store.go

@@ -43,7 +43,8 @@ const (
 	blockSizeConfigKey = "blockSizeInBytes"
 	// blocks must be less than/equal to 100MB in size
 	// ref. https://docs.microsoft.com/en-us/rest/api/storageservices/put-block#uri-parameters
-	defaultBlockSize = 100 * 1024 * 1024
+	maxBlockSize     = 100 * 1024 * 1024
+	defaultBlockSize = 1 * 1024 * 1024
 )
 
 type containerGetter interface {
@@ -259,11 +260,16 @@ func getBlockSize(log logrus.FieldLogger, config map[string]string) int {
 		return defaultBlockSize
 	}
 
-	if blockSize <= 0 || blockSize > defaultBlockSize {
-		log.WithError(err).Warnf("Value provided for config.blockSizeInBytes (%d) is outside the allowed range of 1 to %d, using default block size of %d", blockSize, defaultBlockSize, defaultBlockSize)
+	if blockSize <= 0 {
+		log.WithError(err).Warnf("Value provided for config.blockSizeInBytes (%d) is < 1, using default block size of %d", blockSize, defaultBlockSize)
 		return defaultBlockSize
 	}
 
+	if blockSize > maxBlockSize {
+		log.WithError(err).Warnf("Value provided for config.blockSizeInBytes (%d) is > the max size %d, using max block size of %d", blockSize, maxBlockSize, maxBlockSize)
+		return maxBlockSize
+	}
+
 	return blockSize
 }
 

velero-plugin-for-microsoft-azure/object_store_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -138,3 +139,31 @@ func (m *mockBlob) GetSASURI(ttl time.Duration, sharedKeyCredential *azblob.Shar
 	args := m.Called(ttl, sharedKeyCredential)
 	return args.String(0), args.Error(1)
 }
+
+func TestGetBlockSize(t *testing.T) {
+	logger := logrus.New()
+	config := map[string]string{}
+	// not specified
+	size := getBlockSize(logger, config)
+	assert.Equal(t, defaultBlockSize, size)
+
+	// invalid value specified
+	config[blockSizeConfigKey] = "invalid"
+	size = getBlockSize(logger, config)
+	assert.Equal(t, defaultBlockSize, size)
+
+	// value < 0 specified
+	config[blockSizeConfigKey] = "0"
+	size = getBlockSize(logger, config)
+	assert.Equal(t, defaultBlockSize, size)
+
+	// value > max size specified
+	config[blockSizeConfigKey] = "1048576000"
+	size = getBlockSize(logger, config)
+	assert.Equal(t, maxBlockSize, size)
+
+	// valid value specified
+	config[blockSizeConfigKey] = "1048570"
+	size = getBlockSize(logger, config)
+	assert.Equal(t, 1048570, size)
+}