fix: retries for uploads (#806)

katerina20 · web-flow · commit 407150a7e012 · 2024-05-22T14:24:37.000+03:00
diff --git a/src/main/java/com/crowdin/cli/client/CrowdinClientCore.java b/src/main/java/com/crowdin/cli/client/CrowdinClientCore.java
@@ -77,10 +77,6 @@ protected static <T> List<T> executeRequestFullList(BiFunction<Integer, Integer,
         return directories;
     }
 
-    protected static <T> T executeRequestWithPossibleRetry(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request) throws ResponseException {
-        return executeRequestWithPossibleRetries(errorHandlers, request, 2, defaultMillisToRetry);
-    }
-
     protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request, int maxAttempts, long millisToRetry) throws ResponseException {
         if (maxAttempts < 1) {
             throw new MaxNumberOfRetriesException();
@@ -93,6 +89,7 @@ protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String,
             } catch (InterruptedException ie) {
 //              ignore
             }
+            System.out.printf("Attempting to retry the request due to the error: %s%n", e.getMessage());
             return executeRequestWithPossibleRetries(errorHandlers, request, maxAttempts - 1, millisToRetry);
         }
     }
diff --git a/src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java b/src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java
@@ -1,5 +1,6 @@
 package com.crowdin.cli.client;
 
+import com.crowdin.cli.utils.Utils;
 import com.crowdin.client.branches.model.*;
 import com.crowdin.client.core.model.PatchRequest;
 import com.crowdin.client.labels.model.AddLabelRequest;
@@ -212,10 +213,16 @@ public Long uploadStorage(String fileName, InputStream content) throws  Response
         Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
             put((code, message) -> StringUtils.containsAny(message, "streamIsEmpty", "Stream size is null. Not empty content expected"),
                     new EmptyFileException("Not empty content expected"));
+            put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
+            put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
+            put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
         }};
-        Storage storage = executeRequest(errorHandlers, () -> this.client.getStorageApi()
-            .addStorage(fileName, content)
-            .getData());
+        Storage storage = executeRequestWithPossibleRetries(
+            errorHandlers,
+            () -> this.client.getStorageApi().addStorage(fileName, content).getData(),
+            3,
+            2 * 1000
+        );
         return storage.getId();
     }
 
@@ -244,28 +251,38 @@ public void deleteDirectory(Long directoryId) {
     @Override
     public void updateSource(Long sourceId, UpdateFileRequest request) throws ResponseException {
         Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
-            put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException());
+            put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException("File not found in the storage"));
+            put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
+            put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
+            put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
             put((code, message) -> StringUtils.contains(message, "Invalid SRX specified"), new ResponseException("Invalid SRX file specified"));
             put((code, message) -> code.equals("409"), new FileInUpdateException());
         }};
-        executeRequestWithPossibleRetry(
+        executeRequestWithPossibleRetries(
             errorHandlers,
-            () -> this.client.getSourceFilesApi()
-                .updateOrRestoreFile(this.projectId, sourceId, request));
+            () -> this.client.getSourceFilesApi().updateOrRestoreFile(this.projectId, sourceId, request),
+            3,
+            2 * 1000
+        );
     }
 
     @Override
     public FileInfo addSource(AddFileRequest request) throws ResponseException {
         Map<BiPredicate<String, String>, ResponseException> errorHandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
-            put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException());
+            put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"), new RepeatException("File not found in the storage"));
+            put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
+            put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
+            put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
             put((code, message) -> StringUtils.contains(message, "Name must be unique"), new ExistsResponseException());
             put((code, message) -> StringUtils.contains(message, "Invalid SRX specified"), new ResponseException("Invalid SRX file specified"));
             put((code, message) -> StringUtils.containsAny(message, "isEmpty", "Value is required and can't be empty"), new EmptyFileException("Value is required and can't be empty"));
         }};
-        return executeRequestWithPossibleRetry(
+        return executeRequestWithPossibleRetries(
             errorHandlers,
-            () -> this.client.getSourceFilesApi()
-                .addFile(this.projectId, request).getData());
+            () -> this.client.getSourceFilesApi().addFile(this.projectId, request).getData(),
+            3,
+            2 * 1000
+        );
     }
 
     @Override
@@ -303,12 +320,17 @@ public void uploadTranslations(String languageId, UploadTranslationsRequest requ
             put((code, message) -> code.equals("0") && message.equals("File is not allowed for language"),
                 new WrongLanguageException());
             put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"),
-                new RepeatException());
+                new RepeatException("File not found in the storage"));
+            put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
+            put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
+            put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
         }};
-        executeRequestWithPossibleRetry(
+        executeRequestWithPossibleRetries(
             errorhandlers,
-            () -> this.client.getTranslationsApi()
-                .uploadTranslations(this.projectId, languageId, request));
+            () -> this.client.getTranslationsApi().uploadTranslations(this.projectId, languageId, request),
+            3,
+            2 * 1000
+        );
     }
 
     @Override
@@ -321,13 +343,16 @@ public void uploadTranslationStringsBased(String languageId, UploadTranslationsS
     public ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request) throws ResponseException {
         Map<BiPredicate<String, String>, ResponseException> errorHandler = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
             put((code, message) -> code.equals("409") && message.contains("Another build is currently in progress"),
-                new RepeatException());
+                new RepeatException("Another build is currently in progress"));
+            put((code, message) -> Utils.isServerErrorCode(code), new RepeatException("Server Error"));
+            put((code, message) -> message.contains("Request aborted"), new RepeatException("Request aborted"));
+            put((code, message) -> message.contains("Connection reset"), new RepeatException("Connection reset"));
         }};
         return executeRequestWithPossibleRetries(
             errorHandler,
             () -> this.client.getTranslationsApi().buildProjectTranslation(this.projectId, request).getData(),
             3,
-            60 * 100
+            6 * 1000
         );
     }
 
diff --git a/src/main/java/com/crowdin/cli/client/RepeatException.java b/src/main/java/com/crowdin/cli/client/RepeatException.java
@@ -5,4 +5,8 @@ class RepeatException extends ResponseException {
     public RepeatException() {
         super();
     }
+
+    public RepeatException(String message) {
+        super(message);
+    }
 }
diff --git a/src/main/java/com/crowdin/cli/utils/Utils.java b/src/main/java/com/crowdin/cli/utils/Utils.java
@@ -182,4 +182,14 @@ public static String encodeURL(@NonNull String toEncode) {
     public static String toSingleLineString(String str) {
         return str.replaceAll("[\r\n]+", " ");
     }
+
+    public static boolean isServerErrorCode(String code) {
+        int statusCode;
+        try {
+            statusCode = Integer.parseInt(code);
+        } catch (NumberFormatException e) {
+            return false;
+        }
+        return statusCode >= 500 && statusCode < 600;
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -77,10 +77,6 @@ protected static <T> List<T> executeRequestFullList(BiFunction<Integer, Integer,`
`77`	`77`	`return directories;`
`78`	`78`	`}`
`79`	`79`
`80`		`- protected static <T> T executeRequestWithPossibleRetry(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request) throws ResponseException {`
`81`		`- return executeRequestWithPossibleRetries(errorHandlers, request, 2, defaultMillisToRetry);`
`82`		`- }`
`83`		`-`
`84`	`80`	`protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String, String>, ResponseException> errorHandlers, Supplier<T> request, int maxAttempts, long millisToRetry) throws ResponseException {`
`85`	`81`	`if (maxAttempts < 1) {`
`86`	`82`	`throw new MaxNumberOfRetriesException();`
`@@ -93,6 +89,7 @@ protected static <T> T executeRequestWithPossibleRetries(Map<BiPredicate<String,`
`93`	`89`	`} catch (InterruptedException ie) {`
`94`	`90`	`// ignore`
`95`	`91`	`}`
	`92`	`+ System.out.printf("Attempting to retry the request due to the error: %s%n", e.getMessage());`
`96`	`93`	`return executeRequestWithPossibleRetries(errorHandlers, request, maxAttempts - 1, millisToRetry);`
`97`	`94`	`}`
`98`	`95`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,4 +5,8 @@ class RepeatException extends ResponseException {`
`5`	`5`	`public RepeatException() {`
`6`	`6`	`super();`
`7`	`7`	`}`
	`8`	`+`
	`9`	`+ public RepeatException(String message) {`
	`10`	`+ super(message);`
	`11`	`+ }`
`8`	`12`	`}`