Fix translation update script (#14794)

* Fix translation update script
* Use retry and rate limiter
* Introduce minor delay
* Limit max tokens

---------

Co-authored-by: Mark Sujew <[email protected]>

Author: jfaltermeier

dev-packages/localization-manager/package.json

@@ -36,6 +36,7 @@
     "deepmerge": "^4.2.2",
     "fs-extra": "^4.0.2",
     "glob": "^7.2.0",
+    "limiter": "^2.1.0",
     "tslib": "^2.6.2",
     "typescript": "~5.4.5"
   },

dev-packages/localization-manager/src/deepl-api.ts

@@ -15,10 +15,16 @@
 // *****************************************************************************
 
 import * as bent from 'bent';
+import { RateLimiter } from 'limiter';
 
 const post = bent('POST', 'json', 200);
 // 50 is the maximum amount of translations per request
 const deeplLimit = 50;
+const rateLimiter = new RateLimiter({
+    tokensPerInterval: 10,
+    interval: 'second',
+    fireImmediately: true
+});
 
 export async function deepl(
     parameters: DeeplParameters
@@ -30,12 +36,11 @@ export async function deepl(
     while (textArray.length > 0) {
         textChunks.push(textArray.splice(0, deeplLimit));
     }
-    const responses: DeeplResponse[] = await Promise.all(textChunks.map(chunk => {
+    const responses: DeeplResponse[] = await Promise.all(textChunks.map(async chunk => {
         const parameterCopy: DeeplParameters = { ...parameters, text: chunk };
-        return post(`https://${sub_domain}.deepl.com/v2/translate`, Buffer.from(toFormData(parameterCopy)), {
-            'Content-Type': 'application/x-www-form-urlencoded',
-            'User-Agent': 'Theia-Localization-Manager'
-        });
+        const url = `https://${sub_domain}.deepl.com/v2/translate`;
+        const buffer = Buffer.from(toFormData(parameterCopy));
+        return postWithRetry(url, buffer, 1);
     }));
     const mergedResponse: DeeplResponse = { translations: [] };
     for (const response of responses) {
@@ -47,6 +52,22 @@ export async function deepl(
     return mergedResponse;
 }
 
+async function postWithRetry(url: string, buffer: Buffer, attempt: number): Promise<DeeplResponse> {
+    try {
+        await rateLimiter.removeTokens(Math.min(attempt, 10));
+        const response = await post(url, buffer, {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'User-Agent': 'Theia-Localization-Manager'
+        });
+        return response;
+    } catch (e) {
+        if ('message' in e && typeof e.message === 'string' && e.message.includes('Too Many Requests')) {
+            return postWithRetry(url, buffer, attempt + 1);
+        }
+        throw e;
+    }
+}
+
 /**
  * Coerces the target language into a form expected by Deepl.
  *

package-lock.json

@@ -17,8 +17,7 @@ if (hasNlsFileChanged()) {
 }
 
 function performNlsExtract() {
-    cp.spawnSync('npm', [
-        'run',
+    cp.spawnSync('npx', [
         'theia', 'nls-extract',
         '-o', './packages/core/i18n/nls.json',
         '-e', 'vscode',