Validate size of all uploaded parts per operation

theghost5800 · theghost5800 · commit 8c9298ae859d · 2024-09-04T15:56:17.000+03:00
JIRA:LMCROSSITXSADEPLOY-2948
diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java
@@ -24,7 +24,6 @@
 import org.cloudfoundry.multiapps.controller.core.configuration.Environment;
 import org.cloudfoundry.multiapps.controller.core.health.model.HealthCheckConfiguration;
 import org.cloudfoundry.multiapps.controller.core.health.model.ImmutableHealthCheckConfiguration;
-import org.cloudfoundry.multiapps.controller.persistence.util.Configuration;
 import org.cloudfoundry.multiapps.mta.handlers.ConfigurationParser;
 import org.cloudfoundry.multiapps.mta.model.Platform;
 import org.slf4j.Logger;
@@ -270,10 +269,6 @@ private Set<String> getNotSensitiveConfigVariables() {
                       CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, CFG_SERVICE_HANDLING_MAX_PARALLEL_THREADS);
     }
 
-    public Configuration getFileConfiguration() {
-        return new Configuration(getMaxUploadSize());
-    }
-
     public URL getControllerUrl() {
         if (controllerUrl == null) {
             controllerUrl = getControllerUrlFromEnvironment();
diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/Configuration.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/util/Configuration.java
diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java
@@ -68,6 +68,9 @@ public class Messages {
     public static final String ERROR_WHILE_STARTING_ASYNC_UPLOAD_OF_APP_WITH_NAME_0 = "Error while starting async upload of app with name {0}";
     public static final String REQUIRED_APPLICATION_TO_POLL_0_NOT_FOUND = "Required application to poll: \"{0}\" not found";
     public static final String NOT_INTEGER_PARAMETER_VALUE = "Value \"{0}\" of parameter \"{1}\" is not integer";
+    public static final String ERROR_OCURRED_DURING_VALIDATION_OF_FILES_0 = "Error occurred during validation of files \"{0}\"";
+    public static final String SIZE_OF_ALL_OPERATIONS_FILES_0_EXCEEDS_MAX_UPLOAD_SIZE_1 = "Size of all operation files \"{0}\" exceeds max upload size limit \"{1}\"";
+
     // Audit log messages
 
     // ERROR log messages
diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStep.java
@@ -8,13 +8,15 @@
 import javax.inject.Named;
 
 import org.apache.commons.io.IOUtils;
+import org.cloudfoundry.multiapps.common.ContentException;
 import org.cloudfoundry.multiapps.common.SLException;
 import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor;
 import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
 import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException;
 import org.cloudfoundry.multiapps.controller.process.Messages;
 import org.cloudfoundry.multiapps.controller.process.stream.ArchiveStreamWithName;
+import org.cloudfoundry.multiapps.controller.process.util.FileSweeper;
 import org.cloudfoundry.multiapps.controller.process.util.MergedArchiveStreamCreator;
 import org.cloudfoundry.multiapps.controller.process.variables.Variables;
 import org.springframework.beans.factory.config.BeanDefinition;
@@ -46,6 +48,7 @@ protected String getStepErrorMessage(ProcessContext context) {
 
     private void validateParameters(ProcessContext context) {
         validateExtensionDescriptorFileIds(context);
+        validateFilesSizeLimit(context);
         validateArchive(context);
     }
 
@@ -87,6 +90,40 @@ private void validateDescriptorSize(FileEntry file) {
         }
     }
 
+    private void validateFilesSizeLimit(ProcessContext context) {
+        try {
+            checkFileSizeOfAllFiles(context);
+        } catch (FileStorageException e) {
+            throw new SLException(e, MessageFormat.format(Messages.ERROR_OCURRED_DURING_VALIDATION_OF_FILES_0, e.getMessage()));
+        }
+
+    }
+
+    private void checkFileSizeOfAllFiles(ProcessContext context) throws FileStorageException {
+        long maxFileSizeLimit = configuration.getMaxUploadSize();
+        List<FileEntry> fileEntries = fileService.listFilesBySpaceAndOperationId(context.getVariable(Variables.SPACE_GUID),
+                                                                                 context.getVariable(Variables.CORRELATION_ID));
+        long sizeOfAllFiles = getSizeOfAllFiles(fileEntries);
+        if (sizeOfAllFiles >= maxFileSizeLimit) {
+            deleteFiles(context, fileEntries);
+            throw new ContentException(Messages.SIZE_OF_ALL_OPERATIONS_FILES_0_EXCEEDS_MAX_UPLOAD_SIZE_1, sizeOfAllFiles, maxFileSizeLimit);
+        }
+    }
+
+    private long getSizeOfAllFiles(List<FileEntry> fileEntries) {
+        return fileEntries.stream()
+                          .mapToLong(fileEntry -> fileEntry.getSize()
+                                                           .longValue())
+                          .sum();
+    }
+
+    private void deleteFiles(ProcessContext context, List<FileEntry> fileEntries) throws FileStorageException {
+        FileSweeper fileSweeper = new FileSweeper(context.getVariable(Variables.SPACE_GUID),
+                                                  fileService,
+                                                  context.getVariable(Variables.CORRELATION_ID));
+        fileSweeper.sweep(fileEntries);
+    }
+
     private void validateArchive(ProcessContext context) {
         String[] archivePartIds = getArchivePartIds(context);
         if (archivePartIds.length == 1) {
diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/FileSweeper.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/util/FileSweeper.java
@@ -1,6 +1,7 @@
 package org.cloudfoundry.multiapps.controller.process.util;
 
 import java.text.MessageFormat;
+import java.util.List;
 
 import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileService;
@@ -31,14 +32,24 @@ public void sweep(String fileIds) throws FileStorageException {
         }
     }
 
+    public void sweep(List<FileEntry> fileEntries) throws FileStorageException {
+        for (FileEntry fileEntry : fileEntries) {
+            sweepFileEntry(fileEntry);
+        }
+    }
+
     private void sweepSingle(String fileId) throws FileStorageException {
         FileEntry fileEntry = fileService.getFile(spaceId, fileId);
+        sweepFileEntry(fileEntry);
+    }
+
+    private void sweepFileEntry(FileEntry fileEntry) throws FileStorageException {
         if (operationId.equals(fileEntry.getOperationId())) {
-            fileService.deleteFile(spaceId, fileId);
-            LOGGER.info(MessageFormat.format(Messages.FILE_WITH_ID_0_WAS_DELETED, fileId));
+            fileService.deleteFile(spaceId, fileEntry.getId());
+            LOGGER.info(MessageFormat.format(Messages.FILE_WITH_ID_0_WAS_DELETED, fileEntry.getId()));
             return;
         }
-        LOGGER.warn(MessageFormat.format(Messages.FILE_WITH_ID_0_OPERATION_OWNERSHIP_CHANGED_FROM_0_TO_1, fileId, operationId,
+        LOGGER.warn(MessageFormat.format(Messages.FILE_WITH_ID_0_OPERATION_OWNERSHIP_CHANGED_FROM_0_TO_1, fileEntry.getId(), operationId,
                                          fileEntry.getOperationId()));
     }
 
diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/ValidateDeployParametersStepTest.java
@@ -11,14 +11,14 @@
 import java.nio.file.Paths;
 import java.text.MessageFormat;
 import java.time.Duration;
+import java.util.List;
 import java.util.stream.Stream;
 
 import org.cloudfoundry.multiapps.common.SLException;
 import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;
 import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry;
 import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException;
-import org.cloudfoundry.multiapps.controller.persistence.util.Configuration;
 import org.cloudfoundry.multiapps.controller.process.Messages;
 import org.cloudfoundry.multiapps.controller.process.util.FilePartsMerger;
 import org.cloudfoundry.multiapps.controller.process.variables.Variables;
@@ -35,6 +35,7 @@ class ValidateDeployParametersStepTest extends SyncFlowableStepTest<ValidateDepl
     private static final String EXISTING_BIGGER_FILE_ID = "existingBiggerFileId";
     private static final String NOT_EXISTING_FILE_ID = "notExistingFileId";
     private static final String MERGED_ARCHIVE_NAME = "test-merged";
+    private static final String EXCEEDING_FILE_SIZE_ID = "exceedingFileSizeId";
     private static final String EXCEPTION_START_MESSAGE = "Error validating parameters: ";
 
     private StepInput stepInput;
@@ -63,14 +64,23 @@ private static Stream<Arguments> testExecution() {
 
                          // [4] Process chunked file
                          Arguments.of(new StepInput(MERGED_ARCHIVE_NAME + ".part.0," + MERGED_ARCHIVE_NAME + ".part.1,"
-                             + MERGED_ARCHIVE_NAME + ".part.2", null, 1, VersionRule.HIGHER.toString()), null, true, ""));
+                             + MERGED_ARCHIVE_NAME + ".part.2", null, 1, VersionRule.HIGHER.toString()), null, true, ""),
+
+                         // [5] Max size of entries exceeded
+                         Arguments.of(new StepInput(EXCEEDING_FILE_SIZE_ID + ".part.0," + EXCEEDING_FILE_SIZE_ID + ".part.1,"
+                             + EXCEEDING_FILE_SIZE_ID + ".part.2," + EXCEEDING_FILE_SIZE_ID + ".part.3," + EXCEEDING_FILE_SIZE_ID
+                             + ".part.4", null, 1, VersionRule.HIGHER.toString()),
+                                      MessageFormat.format(Messages.SIZE_OF_ALL_OPERATIONS_FILES_0_EXCEEDS_MAX_UPLOAD_SIZE_1, 5368709120L,
+                                                           4294967296L),
+                                      true, ""));
     }
 
     private static FileEntry createFileEntry(String id, String name, long size) {
         return ImmutableFileEntry.builder()
                                  .id(id)
                                  .name(name)
                                  .size(BigInteger.valueOf(size))
+                                 .operationId("test")
                                  .build();
     }
 
@@ -93,7 +103,7 @@ private void initializeComponents(StepInput stepInput, boolean isArchiveChunked)
         this.stepInput = stepInput;
         this.isArchiveChunked = isArchiveChunked;
         prepareContext();
-        prepareFileService();
+        prepareFileService(stepInput.appArchiveId);
         prepareArchiveMerger();
         prepareConfiguration();
     }
@@ -108,7 +118,7 @@ private void prepareContext() {
         context.setVariable(Variables.MTA_NAMESPACE, "namespace");
     }
 
-    private void prepareFileService() throws FileStorageException {
+    private void prepareFileService(String appArchiveId) throws FileStorageException {
         Mockito.when(fileService.getFile("space-id", EXISTING_FILE_ID))
                .thenReturn(createFileEntry(EXISTING_FILE_ID, "some-file-entry-name", 1024 * 1024L));
         Mockito.when(fileService.getFile("space-id", MERGED_ARCHIVE_NAME + ".part.0"))
@@ -126,6 +136,20 @@ private void prepareFileService() throws FileStorageException {
                .thenReturn(null);
         Mockito.when(fileService.addFile(Mockito.any(FileEntry.class), Mockito.any(InputStream.class)))
                .thenReturn(createFileEntry(EXISTING_FILE_ID, MERGED_ARCHIVE_TEST_MTAR, 1024 * 1024 * 1024L));
+        if (appArchiveId.contains(EXCEEDING_FILE_SIZE_ID)) {
+            List<FileEntry> fileEntries = List.of(createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.0", EXCEEDING_FILE_SIZE_ID + ".part.0",
+                                                                  1024 * 1024 * 1024),
+                                                  createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.1", EXCEEDING_FILE_SIZE_ID + ".part.1",
+                                                                  1024 * 1024 * 1024),
+                                                  createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.2", EXCEEDING_FILE_SIZE_ID + ".part.2",
+                                                                  1024 * 1024 * 1024),
+                                                  createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.3", EXCEEDING_FILE_SIZE_ID + ".part.3",
+                                                                  1024 * 1024 * 1024),
+                                                  createFileEntry(EXCEEDING_FILE_SIZE_ID + ".part.4", EXCEEDING_FILE_SIZE_ID + ".part.4",
+                                                                  1024 * 1024 * 1024));
+            Mockito.when(fileService.listFilesBySpaceAndOperationId(Mockito.anyString(), Mockito.anyString()))
+                   .thenReturn(fileEntries);
+        }
     }
 
     private void prepareArchiveMerger() {
@@ -137,8 +161,8 @@ private void prepareArchiveMerger() {
     private void prepareConfiguration() {
         Mockito.when(configuration.getMaxMtaDescriptorSize())
                .thenReturn(ApplicationConfiguration.DEFAULT_MAX_MTA_DESCRIPTOR_SIZE);
-        Mockito.when(configuration.getFileConfiguration())
-               .thenReturn(new Configuration(ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE));
+        Mockito.when(configuration.getMaxUploadSize())
+               .thenReturn(ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE);
     }
 
     private void validate() {
diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandlerTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/util/OperationInFinalStateHandlerTest.java
@@ -146,7 +146,7 @@ private void prepareFileService(String fileIds, String fileOwnershipProcessId) t
         }
         for (String fileId : fileIds.split(",")) {
             FileEntry entry = ImmutableFileEntry.builder()
-                                                .name(fileId)
+                                                .id(fileId)
                                                 .operationId(fileOwnershipProcessId)
                                                 .build();
             Mockito.when(fileService.getFile(SPACE_ID, fileId))
diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java
@@ -54,7 +54,6 @@
 import org.cloudfoundry.multiapps.controller.persistence.services.AsyncUploadJobService;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileService;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException;
-import org.cloudfoundry.multiapps.controller.persistence.util.Configuration;
 import org.cloudfoundry.multiapps.controller.web.Constants;
 import org.cloudfoundry.multiapps.controller.web.Messages;
 import org.cloudfoundry.multiapps.controller.web.util.SecurityContextUtil;
@@ -372,7 +371,7 @@ private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String
                                 .firstValueAsLong(Constants.CONTENT_LENGTH)
                                 .orElseThrow(() -> new SLException(Messages.FILE_URL_RESPONSE_DID_NOT_RETURN_CONTENT_LENGTH));
 
-        long maxUploadSize = new Configuration().getMaxUploadSize();
+        long maxUploadSize = configuration.getMaxUploadSize();
         if (fileSize > maxUploadSize) {
             throw new SLException(MessageFormat.format(Messages.MAX_UPLOAD_SIZE_EXCEEDED, maxUploadSize));
         }
diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java
@@ -42,7 +42,6 @@
 import org.cloudfoundry.multiapps.controller.persistence.services.AsyncUploadJobService;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileService;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException;
-import org.cloudfoundry.multiapps.controller.persistence.util.Configuration;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -65,7 +64,6 @@
 
 class FilesApiServiceImplTest {
 
-    private static final long MAX_PERMITTED_SIZE = new Configuration().getMaxUploadSize();
     private static final String MTA_ID = "anatz";
     private static final String FILE_URL = Base64.getUrlEncoder()
                                                  .encodeToString("https://host.domain/test.mtar?query=true".getBytes(StandardCharsets.UTF_8));
@@ -100,7 +98,7 @@ protected ResilientOperationExecutor getResilientOperationExecutor() {
     @Mock
     private ExecutorService asyncFileUploadExecutor;
     @Mock
-    private ApplicationConfiguration configuration = new ApplicationConfiguration();
+    private ApplicationConfiguration configuration;
     @Spy
     private DescriptorParserFacadeFactory descriptorParserFactory = new DescriptorParserFacadeFactory(configuration);
     @Mock
@@ -222,6 +220,8 @@ void testUploadFileFromUrl() throws Exception {
                .thenReturn(fileEntry);
         Mockito.when(fileService.getFile(Mockito.eq(SPACE_GUID), Mockito.eq(fileEntry.getId())))
                .thenReturn(fileEntry);
+        Mockito.when(configuration.getMaxUploadSize())
+               .thenReturn(ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE);
         Future<?> future = Mockito.mock(Future.class);
         when(future.isDone()).thenReturn(true);
         prepareAsyncExecutor(future);
@@ -345,7 +345,7 @@ void testUploadFromUrlWhenThereIsValidExistingJob() {
 
     @Test
     void testFileUrlReturnsContentLengthAboveMaxUploadSize() throws Exception {
-        long invalidFileSize = MAX_PERMITTED_SIZE + 1024;
+        long invalidFileSize = ApplicationConfiguration.DEFAULT_MAX_UPLOAD_SIZE + 1024;
         String fileSize = Long.toString(invalidFileSize);
         AsyncUploadJobsQuery query = Mockito.mock(AsyncUploadJobsQuery.class, Answers.RETURNS_SELF);
         String error = "content length exceeds max permitted size of 4GB";

Original file line number	Diff line number	Diff line change
`@@ -146,7 +146,7 @@ private void prepareFileService(String fileIds, String fileOwnershipProcessId) t`
`146`	`146`	`}`
`147`	`147`	`for (String fileId : fileIds.split(",")) {`
`148`	`148`	`FileEntry entry = ImmutableFileEntry.builder()`
`149`		`- .name(fileId)`
	`149`	`+ .id(fileId)`
`150`	`150`	`.operationId(fileOwnershipProcessId)`
`151`	`151`	`.build();`
`152`	`152`	`Mockito.when(fileService.getFile(SPACE_ID, fileId))`