Implement fast incremental compiler

blueprints/ember-cli-typescript/files/tsconfig.json

@@ -20,10 +20,8 @@ inRepoAddons.forEach(function(path) { %>,
       ]
     }
   },
-  "exclude": [
-    "tmp",
-    "dist",
-    "node_modules",
-    "bower_components"
+  "include": [
+    "app",
+    "tests"
   ]
 }

blueprints/ember-cli-typescript/index.js

@@ -1,5 +1,6 @@
 /* eslint-env node */
 
+const { existsSync } = require('fs');
 const path = require('path');
 
 module.exports = {

index.js

@@ -1,41 +1,42 @@
+// @ts-check
 /* eslint-env node */
 
-const fs = require('fs');
-const path = require('path');
-const SilentError = require('silent-error');
-const TsPreprocessor = require('./lib/typescript-preprocessor');
+const IncrementalTypescriptCompiler = require('./lib/incremental-typescript-compiler');
 
 module.exports = {
   name: 'ember-cli-typescript',
 
-  setupPreprocessorRegistry(type, registry) {
-    if (!fs.existsSync(path.join(this.project.root, 'tsconfig.json'))) {
-      // Do nothing; we just won't have the plugin available. This means that if you
-      // somehow end up in a state where it doesn't load, the preprocessor *will*
-      // fail, but this is necessary because the preprocessor depends on packages
-      // which aren't installed until the default blueprint is run
+  included(includer) {
+    this._super.included.apply(this, arguments);
 
-      this.ui.writeInfoLine(
-        'Skipping TypeScript preprocessing as there is no tsconfig.json. ' +
-          '(If this is during installation of the add-on, this is as expected. If it is ' +
-          'while building, serving, or testing the application, this is an error.)'
-      );
-      return;
+    if (includer === this.app) {
+      this.compiler = new IncrementalTypescriptCompiler(this.app, this.project);
+      this.compiler.launch();
     }
+  },
+
+  treeForApp() {
+    if (this.compiler) {
+      let tree = this.compiler.treeForApp();
+      return this._super.treeForApp.call(this, tree);
+    }
+  },
 
-    try {
-      registry.add(
-        'js',
-        new TsPreprocessor({
-          ui: this.ui,
-        })
-      );
-    } catch (ex) {
-      throw new SilentError(
-        `Failed to instantiate TypeScript preprocessor, probably due to an invalid tsconfig.json. Please fix or run \`ember generate ember-cli-typescript\`.\n${
-          ex
-        }`
-      );
+  treeForAddon() {
+    if (this.compiler) {
+      // We manually invoke Babel here rather than calling _super because we're returning
+      // content on behalf of addons that aren't ember-cli-typescript, and the _super impl
+      // would namespace all the files under our own name.
+      let babel = this.project.addons.find(addon => addon.name === 'ember-cli-babel');
+      let tree = this.compiler.treeForAddons();
+      return babel.transpileTree(tree);
     }
   },
+
+  treeForTestSupport() {
+    if (this.compiler) {
+      let tree = this.compiler.treeForTests();
+      return this._super.treeForTestSupport.call(this, tree);
+    }
+  }
 };

lib/incremental-typescript-compiler.js

@@ -0,0 +1,242 @@
+/* eslint-env node */
+
+const execa = require('execa');
+const os = require('os');
+const mkdirp = require('mkdirp');
+const Funnel = require('broccoli-funnel');
+const MergeTrees = require('broccoli-merge-trees');
+const symlinkOrCopy = require('symlink-or-copy');
+const Plugin = require('broccoli-plugin');
+const RSVP = require('rsvp');
+const path = require('path');
+const fs = require('fs');
+const resolve = require('resolve');
+
+const debugCompiler = require('debug')('ember-cli-typescript:compiler');
+const debugAutoresolve = require('debug')('ember-cli-typescript:autoresolve');
+
+module.exports = class IncrementalTypescriptCompiler {
+  constructor(app, project) {
+    if (project._incrementalTsCompiler) {
+      throw new Error(
+        'Multiple IncrementalTypescriptCompiler instances may not be used with the same project.'
+      );
+    }
+
+    project._incrementalTsCompiler = this;
+
+    this.app = app;
+    this.project = project;
+    this.addons = this._discoverAddons(project, []);
+    this.maxBuildCount = 1;
+    this.autoresolveThreshold = 500;
+
+    this._buildDeferred = RSVP.defer();
+    this._isSynced = false;
+    this._triggerDir = `${this.outDir()}/.rebuild`;
+    this._pendingAutoresolve = null;
+    this._didAutoresolve = false;
+  }
+
+  treeForApp() {
+    // This could be more efficient, but we can hopefully assume there won't be dozens
+    // or hundreds of TS addons in dev mode all at once.
+    // Using node-merge-trees in the TypescriptOutput plugin would allow for mappings
+    // like { foo: 'out', bar: 'out' } so we'd only need one Broccoli node for this.
+    let addonAppTrees = this.addons.map(addon => {
+      return new TypescriptOutput(this, {
+        [`${this._relativeAddonRoot(addon)}/app`]: 'app',
+      });
+    });
+
+    let triggerTree = new Funnel(this._triggerDir, { destDir: 'app' });
+
+    let appTree = new TypescriptOutput(this, {
+      [`${this._relativeAppRoot()}/app`]: 'app',
+    });
+
+    let tree = new MergeTrees([...addonAppTrees, triggerTree, appTree], { overwrite: true });
+    return new Funnel(tree, { srcDir: 'app' });
+  }
+
+  treeForAddons() {
+    let paths = {};
+    for (let addon of this.addons) {
+      paths[`${this._relativeAddonRoot(addon)}/addon`] = addon.name;
+    }
+    return new TypescriptOutput(this, paths);
+  }
+
+  treeForTests() {
+    return new TypescriptOutput(this, {
+      [`${this._relativeAppRoot()}/tests`]: 'tests',
+    });
+  }
+
+  buildPromise() {
+    return this._buildDeferred.promise;
+  }
+
+  outDir() {
+    if (!this._outDir) {
+      let outDir = path.join(os.tmpdir(), `e-c-ts-${process.pid}`);
+      this._outDir = outDir;
+      mkdirp.sync(outDir);
+    }
+
+    return this._outDir;
+  }
+
+  launch() {
+    if (!fs.existsSync(`${this.project.root}/tsconfig.json`)) {
+      this.project.ui.writeWarnLine('No tsconfig.json found; skipping TypeScript compilation.');
+      return;
+    }
+
+    mkdirp.sync(this._triggerDir);
+    this._touchRebuildTrigger();
+
+    // argument sequence here is meaningful; don't apply prettier.
+    // prettier-ignore
+    let tsc = execa('tsc', [
+      '--watch',
+      '--outDir', this.outDir(),
+      '--rootDir', this.project.root,
+      '--allowJs', 'false',
+      '--noEmit', 'false',
+    ]);
+
+    tsc.stdout.on('data', data => {
+      this.project.ui.writeLine(
+        data
+          .toString()
+          .trim()
+          .replace(/\u001bc/g, '')
+      );
+
+      if (data.indexOf('Starting incremental compilation') !== -1) {
+        debugCompiler('tsc detected a file change');
+        this.willRebuild();
+        clearTimeout(this._pendingAutoresolve);
+      }
+
+      if (data.indexOf('Compilation complete') !== -1) {
+        debugCompiler('rebuild completed');
+
+        this.didSync();
+
+        if (this._didAutoresolve) {
+          this._touchRebuildTrigger();
+          this.maxBuildCount++;
+        }
+
+        clearTimeout(this._pendingAutoresolve);
+        this._didAutoresolve = false;
+      }
+    });
+
+    tsc.stderr.on('data', data => {
+      this.project.ui.writeError(data.toString().trim());
+    });
+  }
+
+  willRebuild() {
+    if (this._isSynced) {
+      this._isSynced = false;
+      this._buildDeferred = RSVP.defer();
+
+      // Schedule a timer to automatically resolve if tsc doesn't pick up any file changes in a
+      // short period. This may happen if a non-TS file changed, or if the tsc watcher is
+      // drastically behind watchman. If the latter happens, we'll explicitly touch a file in the
+      // broccoli output in order to ensure the changes are picked up.
+      this._pendingAutoresolve = setTimeout(() => {
+        debugAutoresolve('no tsc rebuild; autoresolving...');
+
+        this.didSync();
+        this._didAutoresolve = true;
+      }, this.autoresolveThreshold);
+    }
+  }
+
+  didSync() {
+    this._isSynced = true;
+    this._buildDeferred.resolve();
+  }
+
+  _touchRebuildTrigger() {
+    debugAutoresolve('touching rebuild trigger.');
+    fs.writeFileSync(`${this._triggerDir}/tsc-delayed-rebuild`, '', 'utf-8');
+  }
+
+  _discoverAddons(node, addons) {
+    for (let addon of node.addons) {
+      let devDeps = addon.pkg.devDependencies || {};
+      let deps = addon.pkg.dependencies || {};
+      if (
+        ('ember-cli-typescript' in deps || 'ember-cli-typescript' in devDeps) &&
+        addon.isDevelopingAddon()
+      ) {
+        addons.push(addon);
+      }
+      this._discoverAddons(addon, addons);
+    }
+    return addons;
+  }
+
+  _relativeAppRoot() {
+    // This won't work for apps that have customized their root trees...
+    if (this.app instanceof this.project.require('ember-cli/lib/broccoli/ember-addon')) {
+      return 'tests/dummy';
+    } else {
+      return '.';
+    }
+  }
+
+  _relativeAddonRoot(addon) {
+    let addonRoot = addon.root;
+    if (addonRoot.indexOf(this.project.root) !== 0) {
+      let packagePath = resolve.sync(`${addon.pkg.name}/package.json`, {
+        basedir: this.project.root,
+      });
+      addonRoot = path.dirname(packagePath);
+    }
+
+    return addonRoot.replace(this.project.root, '');
+  }
+};
+
+class TypescriptOutput extends Plugin {
+  constructor(compiler, paths) {
+    super([]);
+    this.compiler = compiler;
+    this.paths = paths;
+    this.buildCount = 0;
+  }
+
+  build() {
+    this.buildCount++;
+
+    // We use this to keep track of the build state between the various
+    // Broccoli trees and tsc; when either tsc or broccoli notices a file
+    // change, we immediately invalidate the previous build output.
+    if (this.buildCount > this.compiler.maxBuildCount) {
+      debugCompiler('broccoli detected a file change');
+      this.compiler.maxBuildCount = this.buildCount;
+      this.compiler.willRebuild();
+    }
+
+    debugCompiler('waiting for tsc output', this.paths);
+    return this.compiler.buildPromise().then(() => {
+      debugCompiler('tsc build complete', this.paths);
+      for (let relativeSrc of Object.keys(this.paths)) {
+        let src = `${this.compiler.outDir()}/${relativeSrc}`;
+        let dest = `${this.outputPath}/${this.paths[relativeSrc]}`;
+        if (fs.existsSync(src)) {
+          symlinkOrCopy.sync(src, dest);
+        } else {
+          mkdirp.sync(dest);
+        }
+      }
+    });
+  }
+}