Parallel VTI diagnostics

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Upgrade `@prettier/plugin-pug` to fix formatter issues. #2347
 - 🙌 Fix collapse code missing end mark. Thanks to contribution from [@yoyo930021](https://github.com/yoyo930021). #2303 and #2352.
+- 🙌 Parallel VTI diagnostics. Thanks to contribution from [@yoyo930021](https://github.com/yoyo930021). #2329.
 
 ### 0.28.0 | 2020-09-23 | [VSIX](https://marketplace.visualstudio.com/_apis/public/gallery/publishers/octref/vsextensions/vetur/0.28.0/vspackage)
 

vti/package.json

@@ -9,6 +9,7 @@
   "dependencies": {
     "chalk": "^4.1.0",
     "glob": "^7.1.2",
+    "jest-worker": "^26.3.0",
     "vls": "^0.5.4",
     "vscode-languageserver": "^6.1.1",
     "vscode-languageserver-protocol": "^3.15.3",

vti/src/cli.ts

@@ -1,132 +1,20 @@
-import {
-  InitializeParams,
-  InitializeRequest,
-  InitializeResult,
-  createProtocolConnection,
-  StreamMessageReader,
-  StreamMessageWriter,
-  Logger,
-  DidOpenTextDocumentNotification,
-  Diagnostic,
-  DiagnosticSeverity,
-  createConnection,
-  ServerCapabilities
-} from 'vscode-languageserver';
-
-import { Duplex } from 'stream';
-import { VLS } from 'vls';
-import { getInitParams } from './initParams';
-import * as fs from 'fs';
-import { URI } from 'vscode-uri';
-import * as glob from 'glob';
 import * as path from 'path';
+import { URI } from 'vscode-uri';
 import * as chalk from 'chalk';
+import Worker from 'jest-worker';
+import { glob } from 'glob';
+import * as os from 'os';
+import { chunk } from 'lodash';
 
-class NullLogger implements Logger {
-  error(_message: string): void {}
-  warn(_message: string): void {}
-  info(_message: string): void {}
-  log(_message: string): void {}
-}
-
-class TestStream extends Duplex {
-  _write(chunk: string, _encoding: string, done: () => void) {
-    this.emit('data', chunk);
-    done();
-  }
-
-  _read(_size: number) {}
-}
-
-async function prepareClientConnection(workspaceUri: URI) {
-  const up = new TestStream();
-  const down = new TestStream();
-  const logger = new NullLogger();
-
-  const clientConnection = createProtocolConnection(new StreamMessageReader(down), new StreamMessageWriter(up), logger);
-
-  const serverConnection = createConnection(new StreamMessageReader(up), new StreamMessageWriter(down));
-  const vls = new VLS(serverConnection as any);
-
-  serverConnection.onInitialize(
-    async (params: InitializeParams): Promise<InitializeResult> => {
-      await vls.init(params);
-
-      console.log('Vetur initialized');
-      console.log('====================================');
-
-      return {
-        capabilities: vls.capabilities as ServerCapabilities
-      };
-    }
-  );
-
-  vls.listen();
-  clientConnection.listen();
-
-  const init = getInitParams(workspaceUri);
-
-  await clientConnection.sendRequest(InitializeRequest.type, init);
-
-  return clientConnection;
-}
-
-async function getDiagnostics(workspaceUri: URI) {
-  const clientConnection = await prepareClientConnection(workspaceUri);
-
-  const files = glob.sync('**/*.vue', { cwd: workspaceUri.fsPath, ignore: ['node_modules/**'] });
-
-  if (files.length === 0) {
-    console.log('No input files');
-    return 0;
-  }
-
-  console.log('');
-  console.log('Getting diagnostics from: ', files, '\n');
+interface VTIProcess { getDiagnostics(workspaceUri: URI, files: string[]): Promise<number>; }
+type VTIWorker = Worker & VTIProcess;
 
-  const absFilePaths = files.map(f => path.resolve(workspaceUri.fsPath, f));
+const SMALL_PROJECT_SIZE = 20;
 
-  let errCount = 0;
-
-  for (const absFilePath of absFilePaths) {
-    await clientConnection.sendNotification(DidOpenTextDocumentNotification.type, {
-      textDocument: {
-        languageId: 'vue',
-        uri: URI.file(absFilePath).toString(),
-        version: 1,
-        text: fs.readFileSync(absFilePath, 'utf-8')
-      }
-    });
-
-    try {
-      const res = (await clientConnection.sendRequest('$/getDiagnostics', {
-        uri: URI.file(absFilePath).toString()
-      })) as Diagnostic[];
-      if (res.length > 0) {
-        console.log(`${chalk.green('File')} : ${chalk.green(absFilePath)}`);
-        res.forEach(d => {
-          /**
-           * Ignore eslint errors for now
-           */
-          if (d.source === 'eslint-plugin-vue') {
-            return;
-          }
-          if (d.severity === DiagnosticSeverity.Error) {
-            console.log(`${chalk.red('Error')}: ${d.message.trim()}`);
-            errCount++;
-          } else {
-            console.log(`${chalk.yellow('Warn')} : ${d.message.trim()}`);
-          }
-        });
-        console.log('');
-      }
-    } catch (err) {
-      console.log(err);
-    }
-  }
-
-  return errCount;
-}
+const getInputGroups = (arr: string[], cpus: number) => {
+  const size = Math.ceil(arr.length / cpus);
+  return chunk(arr, size);
+};
 
 (async () => {
   const myArgs = process.argv.slice(2);
@@ -135,7 +23,7 @@ async function getDiagnostics(workspaceUri: URI) {
   if (myArgs.length > 0 && myArgs[0] === 'diagnostics') {
     console.log('====================================');
     console.log('Getting Vetur diagnostics');
-    let workspaceUri;
+    let workspaceUri: URI;
 
     if (myArgs[1]) {
       const absPath = path.resolve(process.cwd(), myArgs[1]);
@@ -146,7 +34,40 @@ async function getDiagnostics(workspaceUri: URI) {
       workspaceUri = URI.file(process.cwd());
     }
 
-    const errCount = await getDiagnostics(workspaceUri);
+    const files = glob.sync('**/*.vue', { cwd: workspaceUri.fsPath, ignore: ['node_modules/**'] });
+
+    if (files.length === 0) {
+      console.log('No input files');
+      return 0;
+    }
+
+    console.log('');
+    console.log(`Have ${files.length} files.`);
+    console.log('Getting diagnostics from: ', files, '\n');
+
+    const cpus = os.cpus().length - 1;
+
+    const errCount = await (async () => {
+      if (files.length > SMALL_PROJECT_SIZE) {
+        const worker = new Worker(require.resolve('./process'), {
+          numWorkers: cpus,
+          exposedMethods: ['getDiagnostics']
+        }) as VTIWorker;
+
+        worker.getStdout().on('data', data => {
+          process.stdout.write(data.toString('utf-8'));
+        });
+
+        return (
+          await Promise.all(getInputGroups(files, cpus).map(el => worker.getDiagnostics(workspaceUri, el)))
+        ).reduce((sum, el) => sum + el, 0);
+      } else {
+        const process = require('./process') as VTIProcess;
+
+        return process.getDiagnostics(workspaceUri, files);
+      }
+    })();
+
     console.log('====================================');
 
     if (errCount === 0) {

vti/src/process.ts

@@ -0,0 +1,121 @@
+import {
+  InitializeParams,
+  InitializeRequest,
+  InitializeResult,
+  createProtocolConnection,
+  StreamMessageReader,
+  StreamMessageWriter,
+  Logger,
+  DidOpenTextDocumentNotification,
+  Diagnostic,
+  DiagnosticSeverity,
+  createConnection,
+  ServerCapabilities
+} from 'vscode-languageserver';
+
+import { Duplex } from 'stream';
+import { VLS } from 'vls';
+import { getInitParams } from './initParams';
+import * as fs from 'fs';
+import { URI } from 'vscode-uri';
+import * as path from 'path';
+import * as chalk from 'chalk';
+
+class NullLogger implements Logger {
+  error(_message: string): void {}
+  warn(_message: string): void {}
+  info(_message: string): void {}
+  log(_message: string): void {}
+}
+
+class TestStream extends Duplex {
+  _write(chunk: string, _encoding: string, done: () => void) {
+    this.emit('data', chunk);
+    done();
+  }
+
+  _read(_size: number) {}
+}
+
+async function prepareClientConnection(workspaceUri: URI) {
+  const up = new TestStream();
+  const down = new TestStream();
+  const logger = new NullLogger();
+
+  const clientConnection = createProtocolConnection(new StreamMessageReader(down), new StreamMessageWriter(up), logger);
+
+  const serverConnection = createConnection(new StreamMessageReader(up), new StreamMessageWriter(down));
+  const vls = new VLS(serverConnection as any);
+
+  serverConnection.onInitialize(
+    async (params: InitializeParams): Promise<InitializeResult> => {
+      await vls.init(params);
+
+      console.log(`Vetur ${process.env.JEST_WORKER_ID ?? ''} initialized`);
+
+      return {
+        capabilities: vls.capabilities as ServerCapabilities
+      };
+    }
+  );
+
+  vls.listen();
+  clientConnection.listen();
+
+  const init = getInitParams(workspaceUri);
+
+  await clientConnection.sendRequest(InitializeRequest.type, init);
+
+  return clientConnection;
+}
+
+export async function getDiagnostics(workspaceUri: URI, files: string[]) {
+  const clientConnection = await prepareClientConnection(workspaceUri);
+
+  if (files.length === 0) {
+    return 0;
+  }
+
+  const absFilePaths = files.map(f => path.resolve(workspaceUri.fsPath, f));
+
+  let errCount = 0;
+
+  for (const absFilePath of absFilePaths) {
+    await clientConnection.sendNotification(DidOpenTextDocumentNotification.type, {
+      textDocument: {
+        languageId: 'vue',
+        uri: URI.file(absFilePath).toString(),
+        version: 1,
+        text: fs.readFileSync(absFilePath, 'utf-8')
+      }
+    });
+
+    try {
+      const res = (await clientConnection.sendRequest('$/getDiagnostics', {
+        uri: URI.file(absFilePath).toString()
+      })) as Diagnostic[];
+      if (res.length > 0) {
+        console.log(`${chalk.green('File')} : ${chalk.green(absFilePath)}`);
+        res.forEach(d => {
+          /**
+           * Ignore eslint errors for now
+           */
+          if (d.source === 'eslint-plugin-vue') {
+            return;
+          }
+          if (d.severity === DiagnosticSeverity.Error) {
+            console.log(`${chalk.red('Error')}: ${d.message.trim()}`);
+            errCount++;
+          } else {
+            console.log(`${chalk.yellow('Warn')} : ${d.message.trim()}`);
+          }
+        });
+        console.log('');
+      }
+    } catch (err) {
+      console.log(err);
+    }
+  }
+
+  return errCount;
+}

vti/yarn.lock

@@ -2362,6 +2362,15 @@ isobject@^3.0.0, isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8=
 
+jest-worker@^26.3.0:
+  version "26.3.0"
+  resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-26.3.0.tgz#7c8a97e4f4364b4f05ed8bca8ca0c24de091871f"
+  integrity sha512-Vmpn2F6IASefL+DVBhPzI2J9/GJUsqzomdeN+P+dK8/jKxbh8R3BtFnx3FIta7wYlPU62cpJMJQo4kuOowcMnw==
+  dependencies:
+    "@types/node" "*"
+    merge-stream "^2.0.0"
+    supports-color "^7.0.0"
+
 js-beautify@^1.13.0:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/js-beautify/-/js-beautify-1.13.0.tgz#a056d5d3acfd4918549aae3ab039f9f3c51eebb2"
@@ -2696,6 +2705,11 @@ meow@^6.1.1:
     type-fest "^0.13.1"
     yargs-parser "^18.1.3"
 
+merge-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
+  integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
+
 merge2@^1.2.3:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.3.0.tgz#5b366ee83b2f1582c48f87e47cf1a9352103ca81"
@@ -4166,6 +4180,13 @@ supports-color@^6.0.0:
   dependencies:
     has-flag "^3.0.0"
 
+supports-color@^7.0.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da"
+  integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==
+  dependencies:
+    has-flag "^4.0.0"
+
 supports-color@^7.1.0:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.1.0.tgz#68e32591df73e25ad1c4b49108a2ec507962bfd1"