[client-app] Add benchmark mode

Summary:
There are some settings that apply to dev and prod modes that we don't want
to use while benchmarking. E.g. the folder where we store messages,
whether we generate long stack traces in our bluebird promises, etc.
This diff adds a benchmark mode so that we can change these settings to
something that works better for benchmarking.

Test Plan: Run locally, verify it works

Reviewers: evan, juan, spang

Reviewed By: juan, spang

Differential Revision: https://phab.nylas.com/D4374

Author: Mark Hahnenberg

package.json

@@ -45,6 +45,7 @@
     "start": "npm run client",
     "test": "npm run test-client && npm run test-cloud",
     "client": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --dev",
+    "benchmark": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --dev --benchmark",
     "test-client": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --test",
     "test-client-window": "packages/client-app/node_modules/.bin/electron packages/client-app --enable-logging --test=window",
     "test-client-junit": "",

packages/client-app/src/browser/application.es6

@@ -28,7 +28,7 @@ let clipboard = null;
 //
 export default class Application extends EventEmitter {
   async start(options) {
-    const {resourcePath, configDirPath, version, devMode, specMode, safeMode} = options;
+    const {resourcePath, configDirPath, version, devMode, specMode, benchmarkMode, safeMode} = options;
 
     // Initialize GlobalTimer and start timing app boot time
     this.timer = new GlobalTimer()
@@ -39,6 +39,7 @@ export default class Application extends EventEmitter {
     this.configDirPath = configDirPath;
     this.version = version;
     this.devMode = devMode;
+    this.benchmarkMode = benchmarkMode;
     this.specMode = specMode;
     this.safeMode = safeMode;
 
@@ -72,6 +73,7 @@ export default class Application extends EventEmitter {
       configDirPath: this.configDirPath,
       config: this.config,
       devMode: this.devMode,
+      benchmarkMode: this.benchmarkMode,
       specMode: this.specMode,
       safeMode: this.safeMode,
       initializeInBackground: initializeInBackground,

packages/client-app/src/browser/main.js

@@ -23,6 +23,9 @@ const setupConfigDir = (args) => {
   if (args.specMode) {
     defaultDirName = ".nylas-spec";
   }
+  if (args.benchmarkMode) {
+    defaultDirName = ".nylas-bench";
+  }
   let configDirPath = path.join(app.getPath('home'), defaultDirName);
 
   if (args.configDirPath) {
@@ -46,6 +49,7 @@ const setupErrorLogger = (args = {}) => {
   const errorLogger = new ErrorLogger({
     inSpecMode: args.specMode,
     inDevMode: args.devMode,
+    inBenchmarkMode: args.benchmarkMode,
     resourcePath: args.resourcePath,
   });
   process.on('uncaughtException', errorLogger.reportError);
@@ -58,6 +62,7 @@ const declareOptions = (argv) => {
   const options = optimist(argv);
   options.usage("Nylas Mail v" + (app.getVersion()) + "\n\nUsage: nylas-mail [options]\n\nRun Nylas Mail: The open source extensible email client\n\n`nylas-mail --dev` to start the client in dev mode.\n\n`n1 --test` to run unit tests.");
   options.alias('d', 'dev').boolean('d').describe('d', 'Run in development mode.');
+  options.alias('b', 'benchmark').boolean('b').describe('b', 'Run in benchmark mode.');
   options.alias('t', 'test').boolean('t').describe('t', 'Run the specified specs and exit with error code on failures.');
   options.boolean('safe').describe('safe', 'Do not load packages from ~/.nylas-mail/packages or ~/.nylas/dev/packages.');
   options.alias('h', 'help').boolean('h').describe('h', 'Print this usage message.');
@@ -83,7 +88,8 @@ const parseCommandLine = (argv) => {
     process.stdout.write(version + "\n");
     process.exit(0);
   }
-  const devMode = args['dev'] || args['test'];
+  const devMode = args['dev'] || args['test'] || args['benchmark'];
+  const benchmarkMode = args['benchmark'];
   const logFile = args['log-file'];
   const specMode = args['test'];
   const jUnitXmlPath = args['junit-xml'];
@@ -131,6 +137,7 @@ const parseCommandLine = (argv) => {
     background,
     logFile,
     specMode,
+    benchmarkMode,
     jUnitXmlPath,
     safeMode,
     configDirPath,

packages/client-app/src/browser/nylas-window.coffee

@@ -28,6 +28,7 @@ class NylasWindow
      pathToOpen,
      @isSpec,
      @devMode,
+     @benchmarkMode,
      @windowKey,
      @safeMode,
      @neverClose,
@@ -83,6 +84,7 @@ class NylasWindow
     loadSettings.appVersion = app.getVersion()
     loadSettings.resourcePath = @resourcePath
     loadSettings.devMode ?= false
+    loadSettings.benchmarkMode ?= false
     loadSettings.safeMode ?= false
     loadSettings.mainWindow ?= @mainWindow
     loadSettings.windowType ?= "default"

packages/client-app/src/browser/window-manager.es6

@@ -10,15 +10,15 @@ const ONBOARDING_WINDOW = "onboarding"
 
 export default class WindowManager {
 
-  constructor({devMode, safeMode, specMode, resourcePath, configDirPath, initializeInBackground, config}) {
+  constructor({devMode, benchmarkMode, safeMode, specMode, resourcePath, configDirPath, initializeInBackground, config}) {
     this.initializeInBackground = initializeInBackground;
     this._windows = {};
 
     const onCreatedHotWindow = (win) => {
       this._registerWindow(win);
       this._didCreateNewWindow(win);
     }
-    this.windowLauncher = new WindowLauncher({devMode, safeMode, specMode, resourcePath, configDirPath, config, onCreatedHotWindow});
+    this.windowLauncher = new WindowLauncher({devMode, benchmarkMode, safeMode, specMode, resourcePath, configDirPath, config, onCreatedHotWindow});
   }
 
   get(windowKey) {
@@ -227,6 +227,7 @@ export default class WindowManager {
       hidden: true,
       isSpec: true,
       devMode: true,
+      benchmarkMode: false,
       toolbar: false,
     }
 

packages/client-app/src/nylas-env.es6

@@ -174,7 +174,7 @@ export default class NylasEnvConstructor {
     const ActionBridge = require('./flux/action-bridge').default;
     const MenuManager = require('./menu-manager').default;
 
-    const {devMode, safeMode, resourcePath, configDirPath, windowType} = this.getLoadSettings();
+    const {devMode, benchmarkMode, safeMode, resourcePath, configDirPath, windowType} = this.getLoadSettings();
 
     document.body.classList.add(`platform-${process.platform}`);
     document.body.classList.add(`window-type-${windowType}`);
@@ -209,7 +209,7 @@ export default class NylasEnvConstructor {
     const specMode = this.inSpecMode();
 
     this.commands = new CommandRegistry();
-    this.packages = new PackageManager({devMode, configDirPath, resourcePath, safeMode, specMode});
+    this.packages = new PackageManager({devMode, benchmarkMode, configDirPath, resourcePath, safeMode, specMode});
     this.styles = new StyleManager();
     document.head.appendChild(new StylesElement());
     this.themes = new ThemeManager({packageManager: this.packages, configDirPath, resourcePath, safeMode});
@@ -296,7 +296,7 @@ export default class NylasEnvConstructor {
       return this.reportError(error);
     });
 
-    if (this.inSpecMode() || this.inDevMode()) {
+    if (this.inSpecMode() || (this.inDevMode() && !this.inBenchmarkMode())) {
       return Promise.config({longStackTraces: true});
     }
     return null;
@@ -449,6 +449,10 @@ export default class NylasEnvConstructor {
     return this.getLoadSettings().devMode;
   }
 
+  inBenchmarkMode() {
+    return this.getLoadSettings().benchmarkMode;
+  }
+
   // Public: Is the current window in safe mode?
   inSafeMode() {
     return this.getLoadSettings().safeMode;

packages/client-sync/src/local-sync-worker/sync-worker.es6

@@ -263,6 +263,14 @@ class SyncWorker {
       onConnected: async ([listenerConn], onDone) => {
         this._mailListenerIMAPConn = listenerConn;
         this._mailListenerIMAPConn._db = this._db;
+        this._mailListenerIMAPConnDisposer = onDone
+
+        // We don't want to listen for new mail while benchmarking initial
+        // sync b/c receiving new mail can significantly increase the variance
+        // in our measurements.
+        if (NylasEnv.inBenchmarkMode()) {
+          return true;
+        }
 
         this._mailListenerIMAPConn.on('mail', () => {
           this._onInboxUpdates(`You've got mail`);
@@ -275,9 +283,6 @@ class SyncWorker {
           if (this._shouldIgnoreInboxFlagUpdates) { return; }
           this._onInboxUpdates(`There are flag updates on the inbox`);
         })
-
-        this._mailListenerIMAPConnDisposer = onDone
-        // Return true to keep connection open
         return true
       },
     })

scripts/benchmark-initial-sync.sh

@@ -22,7 +22,7 @@ then
 fi
 
 CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-NYLAS_DIR="$HOME/.nylas-dev"
+NYLAS_DIR="$HOME/.nylas-bench"
 EDGEHILL_DB="$NYLAS_DIR/edgehill.db"
 TIME_LIMIT=120
 ITERS=5
@@ -31,7 +31,7 @@ for i in `seq 1 $ITERS`
 do
   bash $CWD/drop-data-except-accounts.sh > /dev/null
 
-  (npm start &> /dev/null &)
+  (npm run benchmark &> /dev/null &)
 
   sleep $TIME_LIMIT
 

scripts/drop-data-except-accounts.sh

@@ -3,7 +3,7 @@
 set -e
 
 CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-NYLAS_DIR="$HOME/.nylas-dev"
+NYLAS_DIR="$HOME/.nylas-bench"
 TRUNCATE_TABLES="
 Account 
 AccountPluginMetadata