Skip to content
This repository was archived by the owner on Feb 1, 2023. It is now read-only.

Commit c14dbf6

Browse files
ArtsiomChArtsiomCh
authored
Merge pull request #38 from snyk/fix/re-try_for_most_failed_api_calls
fix: make re-try attempts for most api calls if not succeed for any reason (except 401 - auth failed) [ROAD-808] [ROAD-528]
2 parents 4d21c1f + f73a24e commit c14dbf6

File tree

5 files changed

+217
-62
lines changed

5 files changed

+217
-62
lines changed

‎CHANGELOG.md

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
## [2.3.2] - 2022-05
2+
- fix: make re-try attempts for most api calls if not succeed for any reason (except 401 - auth failed)
3+
14
## [2.3.1] - 2022-04
25
- fix: adjust max file size to the correct size of 1MB (previously: 4000 bytes)
36
- chore: update upload attemps to 10

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ plugins {
66

77
group = "io.snyk.code.sdk"
88
archivesBaseName = "snyk-code-client"
9-
version = "2.3.1"
9+
version = "2.3.2"
1010

1111
repositories {
1212
mavenLocal()

‎src/main/java/ai/deepcode/javaclient/core/AnalysisDataBase.java

+114-54
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public void removeFilesFromCache(@NotNull Collection<Object> files) {
131131
"Request to remove from cache " + files.size() + " files: " + first50FilesName);
132132
// todo: do we really need mutex here?
133133
MUTEX.lock();
134-
dcLogger.logInfo("MUTEX LOCK");
134+
dcLogger.logInfo("MUTEX LOCK, hold count = " + MUTEX.getHoldCount());
135135
int removeCounter = 0;
136136
for (Object file : files) {
137137
if (file != null && isFileInCache(file)) {
@@ -149,8 +149,8 @@ public void removeFilesFromCache(@NotNull Collection<Object> files) {
149149
+ " files. Were not in cache: "
150150
+ (files.size() - removeCounter));
151151
} finally {
152-
dcLogger.logInfo("MUTEX RELEASED");
153152
MUTEX.unlock();
153+
dcLogger.logInfo("MUTEX RELEASED, hold count = " + MUTEX.getHoldCount());
154154
}
155155
updateUIonFilesRemovalFromCache(files);
156156
}
@@ -229,7 +229,7 @@ public void updateCachedResultsForFiles(
229229
}
230230
try {
231231
MUTEX.lock();
232-
dcLogger.logInfo("MUTEX LOCK");
232+
dcLogger.logInfo("MUTEX LOCK, hold count = " + MUTEX.getHoldCount());
233233
setUpdateInProgress(project);
234234
Collection<Object> filesToProceed =
235235
allProjectFiles.stream()
@@ -256,39 +256,71 @@ public void updateCachedResultsForFiles(
256256
dcLogger.logInfo("Files to remove: " + filesToRemove.size() + " files: " + filesToRemove);
257257
}
258258
mapFile2Suggestions.putAll(
259-
retrieveSuggestions(project, filesToProceed, filesToRemove, progress));
259+
retrieveSuggestions(project, filesToProceed, filesToRemove, progress)
260+
);
261+
} catch (BundleIdExpire404Exception e) {
262+
// re-try to create bundle from scratch
263+
retryFullUpdateCachedResults(project, allProjectFiles, progress, 2);
260264
} finally {
261-
// if (filesToProceed != null && !filesToProceed.isEmpty())
262-
dcLogger.logInfo("MUTEX RELEASED");
263265
MUTEX.unlock();
266+
dcLogger.logInfo("MUTEX RELEASED, hold count = " + MUTEX.getHoldCount());
264267
unsetUpdateInProgress(project);
265268
pdUtils.refreshPanel(project);
266-
// ServiceManager.getService(project, myTodoView.class).refresh();
267269
}
268270
}
269271

272+
private void retryFullUpdateCachedResults(
273+
@NotNull Object project,
274+
@NotNull Collection<Object> allProjectFiles,
275+
@NotNull Object progress,
276+
int attemptCounter
277+
) {
278+
if (attemptCounter <= 0) {
279+
showWarnIfNeeded(project, "Operations with bundle failed. Please try again later or contact Snyk support");
280+
return;
281+
}
282+
removeProjectFromCaches(project);
283+
try {
284+
mapFile2Suggestions.putAll(
285+
retrieveSuggestions(project, allProjectFiles, Collections.emptyList(), progress)
286+
);
287+
} catch (BundleIdExpire404Exception ex) {
288+
retryFullUpdateCachedResults(project, allProjectFiles, progress, attemptCounter - 1);
289+
}
290+
}
291+
292+
private static class TokenInvalid401Exception extends Exception {}
293+
294+
private static class BundleIdExpire404Exception extends Exception {}
295+
296+
private static class ApiCallNotSucceedException extends Exception {}
297+
270298
private static final Set<Object> projectsLoginRequested = ConcurrentHashMap.newKeySet();
271299
private static final Set<Object> projectsWithNotSucceedWarnShown = ConcurrentHashMap.newKeySet();
272300

273-
private boolean isNotSucceed(
274-
@NotNull Object project, EmptyResponse response, String internalMessage) {
301+
private void showWarnIfNeeded(@NotNull Object project, @NotNull String message) {
302+
pdUtils.showWarn(message, project, !projectsWithNotSucceedWarnShown.add(project));
303+
}
304+
305+
private void checkApiCallSucceed(
306+
@NotNull Object project, EmptyResponse response, String internalMessage
307+
) throws ApiCallNotSucceedException, TokenInvalid401Exception, BundleIdExpire404Exception {
275308
if (response.getStatusCode() == 200) {
276309
projectsWithNotSucceedWarnShown.remove(project);
277310
projectsLoginRequested.remove(project);
278311
} else {
279312
final String fullLogMessage =
280313
internalMessage + response.getStatusCode() + " " + response.getStatusDescription();
281314
dcLogger.logWarn(fullLogMessage);
282-
final boolean wasWarnShown = projectsWithNotSucceedWarnShown.contains(project);
283315
if (response.getStatusCode() == 401) {
284-
pdUtils.isLogged(project, !projectsLoginRequested.contains(project));
285-
projectsLoginRequested.add(project);
316+
pdUtils.isLogged(project, !projectsLoginRequested.add(project));
317+
throw new TokenInvalid401Exception();
318+
} else if (response.getStatusCode() == 404) {
319+
throw new BundleIdExpire404Exception();
286320
} else {
287-
pdUtils.showWarn(response.getStatusDescription(), project, wasWarnShown);
321+
throw new ApiCallNotSucceedException();
288322
}
289-
projectsWithNotSucceedWarnShown.add(project);
290323
}
291-
return response.getStatusCode() != 200;
292324
}
293325

294326
static final int MAX_BUNDLE_SIZE = 4000000; // bytes
@@ -299,22 +331,31 @@ private Map<Object, List<SuggestionForFile>> retrieveSuggestions(
299331
@NotNull Object project,
300332
@NotNull Collection<Object> filesToProceed,
301333
@NotNull Collection<Object> filesToRemove,
302-
@NotNull Object progress) {
334+
@NotNull Object progress
335+
) throws BundleIdExpire404Exception {
303336
if (filesToProceed.isEmpty() && filesToRemove.isEmpty()) {
304337
dcLogger.logWarn("Both filesToProceed and filesToRemove are empty");
305338
return EMPTY_MAP;
306339
}
307340
// no needs to check login here as it will be checked anyway during every api response's check
308341
// if (!LoginUtils.isLogged(project, false)) return EMPTY_MAP;
309342

310-
List<String> missingFiles = createBundleStep(project, filesToProceed, filesToRemove, progress);
343+
List<String> missingFiles = null;
344+
try {
345+
missingFiles = createBundleStep(project, filesToProceed, filesToRemove, progress);
346+
} catch (ApiCallNotSucceedException e) {
347+
// re-try createBundleStep from scratch for few times, i.e. do the same as if parent bundle is expired
348+
mapProject2BundleId.put(project, "");
349+
throw new BundleIdExpire404Exception();
350+
} catch (TokenInvalid401Exception e) {
351+
return EMPTY_MAP;
352+
}
311353

312354
if (filesToProceed.isEmpty()) { // no sense to proceed
313355
return EMPTY_MAP;
314356
}
315357
boolean filesUploaded = uploadFilesStep(project, filesToProceed, missingFiles, progress);
316358
if (!filesUploaded) { // no sense to proceed
317-
dcLogger.logWarn("Files upload FAIL");
318359
return EMPTY_MAP;
319360
}
320361

@@ -328,8 +369,12 @@ private Map<Object, List<SuggestionForFile>> retrieveSuggestions(
328369
pdUtils.progressCheckCanceled(progress);
329370
List<String> filesToAnalyse =
330371
filesToProceed.stream().map(pdUtils::getDeepCodedFilePath).collect(Collectors.toList());
331-
GetAnalysisResponse getAnalysisResponse =
332-
doGetAnalysis(project, bundleId, progress, filesToAnalyse);
372+
GetAnalysisResponse getAnalysisResponse;
373+
try {
374+
getAnalysisResponse = doGetAnalysis(project, bundleId, progress, filesToAnalyse);
375+
} catch (TokenInvalid401Exception e) {
376+
return EMPTY_MAP;
377+
}
333378
Map<Object, List<SuggestionForFile>> result =
334379
parseGetAnalysisResponse(project, filesToProceed, getAnalysisResponse, progress);
335380
dcLogger.logInfo(
@@ -346,7 +391,8 @@ private List<String> createBundleStep(
346391
@NotNull Object project,
347392
@NotNull Collection<Object> filesToProceed,
348393
@NotNull Collection<Object> filesToRemove,
349-
@NotNull Object progress) {
394+
@NotNull Object progress
395+
) throws BundleIdExpire404Exception, ApiCallNotSucceedException, TokenInvalid401Exception {
350396
long startTime = System.currentTimeMillis();
351397
pdUtils.progressSetText(progress, PREPARE_FILES_TEXT);
352398
dcLogger.logInfo(PREPARE_FILES_TEXT);
@@ -401,7 +447,8 @@ private boolean uploadFilesStep(
401447
@NotNull Object project,
402448
@NotNull Collection<Object> filesToProceed,
403449
@NotNull List<String> missingFiles,
404-
@NotNull Object progress) {
450+
@NotNull Object progress
451+
) throws BundleIdExpire404Exception {
405452
long startTime = System.currentTimeMillis();
406453
pdUtils.progressSetText(progress, UPLOADING_FILES_TEXT);
407454
pdUtils.progressCheckCanceled(progress);
@@ -424,11 +471,21 @@ private boolean uploadFilesStep(
424471
+ " more times:\nmissingFiles = "
425472
+ missingFiles);
426473
}
427-
uploadFiles(project, filesToProceed, missingFiles, bundleId, progress);
428-
List<String> newMissingFiles = checkBundle(project, bundleId);
429-
missingFiles = (newMissingFiles != null) ? newMissingFiles : missingFiles;
474+
List<String> newMissingFiles;
475+
try {
476+
uploadFiles(project, filesToProceed, missingFiles, bundleId, progress);
477+
newMissingFiles = checkBundle(project, bundleId);
478+
} catch (TokenInvalid401Exception e) {
479+
break;
480+
} catch (ApiCallNotSucceedException e) {
481+
newMissingFiles = missingFiles;
482+
}
483+
missingFiles = newMissingFiles;
430484
counter++;
431485
}
486+
if (counter >= attempts) {
487+
showWarnIfNeeded(project, "Failed to upload files. Please try again later or contact Snyk support");
488+
}
432489
}
433490
dcLogger.logInfo(
434491
"--- Upload Files took: " + (System.currentTimeMillis() - startTime) + " milliseconds");
@@ -440,7 +497,8 @@ private void uploadFiles(
440497
@NotNull Collection<Object> filesToProceed,
441498
@NotNull List<String> missingFiles,
442499
@NotNull String bundleId,
443-
@NotNull Object progress) {
500+
@NotNull Object progress
501+
) throws ApiCallNotSucceedException, TokenInvalid401Exception, BundleIdExpire404Exception {
444502
Map<String, Object> mapPath2File =
445503
filesToProceed.stream().collect(Collectors.toMap(pdUtils::getDeepCodedFilePath, it -> it));
446504
int fileCounter = 0;
@@ -487,20 +545,21 @@ private void uploadFiles(
487545
*
488546
* @return list of the current missingFiles or NULL if not succeed.
489547
*/
490-
@Nullable
491-
private List<String> checkBundle(@NotNull Object project, @NotNull String bundleId) {
548+
private List<String> checkBundle(
549+
@NotNull Object project,
550+
@NotNull String bundleId
551+
) throws TokenInvalid401Exception, BundleIdExpire404Exception, ApiCallNotSucceedException {
492552
CreateBundleResponse checkBundleResponse =
493553
restApi.checkBundle(deepCodeParams.getSessionToken(), bundleId);
494-
if (isNotSucceed(project, checkBundleResponse, "Bad CheckBundle request: ")) {
495-
return null;
496-
}
554+
checkApiCallSucceed(project, checkBundleResponse, "Bad CheckBundle request: ");
497555
return checkBundleResponse.getMissingFiles();
498556
}
499557

500558
private CreateBundleResponse makeNewBundle(
501559
@NotNull Object project,
502560
@NotNull FileHashRequest request,
503-
@NotNull Collection<Object> filesToRemove) {
561+
@NotNull Collection<Object> filesToRemove
562+
) throws BundleIdExpire404Exception, ApiCallNotSucceedException, TokenInvalid401Exception {
504563
final String parentBundleId = mapProject2BundleId.getOrDefault(project, "");
505564
if (!parentBundleId.isEmpty()
506565
&& !filesToRemove.isEmpty()
@@ -540,19 +599,17 @@ private CreateBundleResponse makeNewBundle(
540599
"/DEEPCODE_PRIVATE_BUNDLE/0000000000000000000000000000000000000000000000000000000000000000")) {
541600
newBundleId = "";
542601
}
602+
checkApiCallSucceed(project, bundleResponse, "Bad Create/Extend Bundle request: ");
543603
mapProject2BundleId.put(project, newBundleId);
544-
isNotSucceed(project, bundleResponse, "Bad Create/Extend Bundle request: ");
545-
// just make new bundle in case of 404 Parent bundle has expired
546-
return (bundleResponse.getStatusCode() == 404)
547-
? makeNewBundle(project, request, filesToRemove)
548-
: bundleResponse;
604+
return bundleResponse;
549605
}
550606

551607
private void doUploadFiles(
552608
@NotNull Object project,
553609
@NotNull Collection<Object> psiFiles,
554610
@NotNull String bundleId,
555-
@NotNull Object progress) {
611+
@NotNull Object progress
612+
) throws ApiCallNotSucceedException, TokenInvalid401Exception, BundleIdExpire404Exception {
556613
dcLogger.logInfo("Uploading " + psiFiles.size() + " files... ");
557614
if (psiFiles.isEmpty()) return;
558615

@@ -571,18 +628,19 @@ private void doUploadFiles(
571628
deepCodeParams.getSessionToken(),
572629
bundleId,
573630
new ExtendBundleWithContentRequest(files, Collections.emptyList()));
574-
isNotSucceed(project, uploadFilesResponse, "Bad UploadFiles request: ");
631+
checkApiCallSucceed(project, uploadFilesResponse, "Bad UploadFiles request: ");
575632
}
576633

577634
@NotNull
578635
private GetAnalysisResponse doGetAnalysis(
579636
@NotNull Object project,
580637
@NotNull String bundleId,
581638
@NotNull Object progress,
582-
List<String> filesToAnalyse) {
639+
List<String> filesToAnalyse
640+
) throws TokenInvalid401Exception {
583641
GetAnalysisResponse response;
584642
int counter = 0;
585-
int failWith404counts = 0;
643+
int skippedFailsCount = 0;
586644
final long timeout = deepCodeParams.getTimeoutForGettingAnalysesMs();
587645
final long attempts = timeout / PlatformDependentUtilsBase.DEFAULT_DELAY;
588646
final long endTime = System.currentTimeMillis() + timeout;
@@ -600,16 +658,18 @@ private GetAnalysisResponse doGetAnalysis(
600658

601659
pdUtils.progressCheckCanceled(progress);
602660
dcLogger.logInfo(response.toString());
603-
if (isNotSucceed(project, response, "Bad GetAnalysis request: ")) {
604-
if (response.getStatusCode() != 404 || failWith404counts >= 5) {
661+
662+
try {
663+
checkApiCallSucceed(project, response, "Bad GetAnalysis request: ");
664+
skippedFailsCount = 0;
665+
} catch (BundleIdExpire404Exception | ApiCallNotSucceedException e) {
666+
if (skippedFailsCount >= 5) {
667+
showWarnIfNeeded(project, "Failed to get analysis results. Please try again later or contact Snyk support");
605668
return new GetAnalysisResponse();
606669
} else {
607-
failWith404counts++;
670+
skippedFailsCount++;
608671
}
609-
} else {
610-
failWith404counts = 0;
611672
}
612-
613673
double responseProgress = response.getProgress();
614674
if (responseProgress <= 0 || responseProgress > 1) {
615675
responseProgress = ((double) counter) / attempts;
@@ -620,13 +680,13 @@ private GetAnalysisResponse doGetAnalysis(
620680

621681
if (System.currentTimeMillis() >= endTime) {
622682
dcLogger.logWarn("Timeout expire for waiting analysis results.");
623-
pdUtils.showWarn(
624-
"Can't get analysis results from the server. Timeout of "
625-
+ timeout / 1000
626-
+ " sec. is reached."
627-
+ " Please, increase timeout or try again later.",
628-
project,
629-
false);
683+
showWarnIfNeeded(
684+
project,
685+
"Can't get analysis results from the server. Timeout of "
686+
+ timeout / 1000
687+
+ " sec. is reached."
688+
+ " Please, increase timeout or try again later."
689+
);
630690
break;
631691
}
632692

0 commit comments

Comments
 (0)