Improvements to bundling. (#3965)

Moves to using a minimal System loader for bundles generated by Deno.
TypeScript in 3.8 will be able to output TLA for modules, and the loader
is written to take advantage of that as soon as we update Deno to TS
3.8.

System also allows us to support `import.meta` and provide more ESM
aligned assignment of exports, as well as there is better handling of
circular imports.

The loader is also very terse versus to try to save overhead.

Also, fixed an issue where abstract classes were not being re-exported.

Fixes #2553
Fixes #3559
Fixes #3751
Fixes #3825
Refs #3301

Author: kitsonk

cli/js/compiler_api_test.ts

@@ -78,15 +78,15 @@ test(async function bundleApiSources() {
     "/bar.ts": `export const bar = "bar";\n`
   });
   assert(diagnostics == null);
-  assert(actual.includes(`instantiate("foo")`));
-  assert(actual.includes(`__rootExports["bar"]`));
+  assert(actual.includes(`__inst("foo")`));
+  assert(actual.includes(`__exp["bar"]`));
 });
 
 test(async function bundleApiNoSources() {
   const [diagnostics, actual] = await bundle("./cli/tests/subdir/mod1.ts");
   assert(diagnostics == null);
-  assert(actual.includes(`instantiate("mod1")`));
-  assert(actual.includes(`__rootExports["printHello3"]`));
+  assert(actual.includes(`__inst("mod1")`));
+  assert(actual.includes(`__exp["printHello3"]`));
 });
 
 test(async function bundleApiConfig() {

cli/js/compiler_bootstrap.ts

@@ -55,4 +55,4 @@ export const TS_SNAPSHOT_PROGRAM = ts.createProgram({
  * We read all static assets during the snapshotting process, which is
  * why this is located in compiler_bootstrap.
  */
-export const BUNDLE_LOADER = getAsset("bundle_loader.js");
+export const SYSTEM_LOADER = getAsset("system_loader.js");

cli/js/compiler_bundler.ts

@@ -1,6 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { BUNDLE_LOADER } from "./compiler_bootstrap.ts";
+import { SYSTEM_LOADER } from "./compiler_bootstrap.ts";
 import {
   assert,
   commonPath,
@@ -44,18 +44,18 @@ export function buildBundle(
     .replace(/\.\w+$/i, "");
   let instantiate: string;
   if (rootExports && rootExports.length) {
-    instantiate = `const __rootExports = instantiate("${rootName}");\n`;
+    instantiate = `const __exp = await __inst("${rootName}");\n`;
     for (const rootExport of rootExports) {
       if (rootExport === "default") {
-        instantiate += `export default __rootExports["${rootExport}"];\n`;
+        instantiate += `export default __exp["${rootExport}"];\n`;
       } else {
-        instantiate += `export const ${rootExport} = __rootExports["${rootExport}"];\n`;
+        instantiate += `export const ${rootExport} = __exp["${rootExport}"];\n`;
       }
     }
   } else {
-    instantiate = `instantiate("${rootName}");\n`;
+    instantiate = `await __inst("${rootName}");\n`;
   }
-  return `${BUNDLE_LOADER}\n${data}\n${instantiate}`;
+  return `${SYSTEM_LOADER}\n${data}\n${instantiate}`;
 }
 
 /** Set the rootExports which will by the `emitBundle()` */
@@ -80,6 +80,7 @@ export function setRootExports(program: ts.Program, rootModule: string): void {
     // out, so inspecting SymbolFlags that might be present that are type only
     .filter(
       sym =>
+        sym.flags & ts.SymbolFlags.Class ||
         !(
           sym.flags & ts.SymbolFlags.Interface ||
           sym.flags & ts.SymbolFlags.TypeLiteral ||

cli/js/compiler_host.ts

@@ -34,7 +34,7 @@ export const ASSETS = "$asset$";
  * runtime). */
 export const defaultBundlerOptions: ts.CompilerOptions = {
   inlineSourceMap: false,
-  module: ts.ModuleKind.AMD,
+  module: ts.ModuleKind.System,
   outDir: undefined,
   outFile: `${OUT_DIR}/bundle.js`,
   // disabled until we have effective way to modify source maps

cli/tests/bundle.test.out

@@ -1,22 +1,24 @@
 [WILDCARD]
-let define;
+let System;
+let __inst;
 [WILDCARD]
-let instantiate;
-[WILDCARD]
-(function() {
+(() => {
 [WILDCARD]
 })();
 
-define("print_hello", ["require", "exports"], function (require, exports) {
+System.register("print_hello", [], function (exports_1, context_1) {
 [WILDCARD]
 });
-define("mod1", ["require", "exports", "subdir2/mod2"], function (require, exports, mod2_ts_1) {
+System.register("subdir2/mod2", ["print_hello"], function (exports_2, context_2) {
+[WILDCARD]
+});
+System.register("mod1", ["subdir2/mod2"], function (exports_3, context_3) {
 [WILDCARD]
 });
 
-const __rootExports = instantiate("mod1");
-export const returnsHi = __rootExports["returnsHi"];
-export const returnsFoo2 = __rootExports["returnsFoo2"];
-export const printHello3 = __rootExports["printHello3"];
-export const throwsError = __rootExports["throwsError"];
-
+const __exp = await __inst("mod1");
+export const returnsHi = __exp["returnsHi"];
+export const returnsFoo2 = __exp["returnsFoo2"];
+export const printHello3 = __exp["printHello3"];
+export const throwsError = __exp["throwsError"];
+[WILDCARD]

cli/tests/integration_tests.rs

@@ -252,6 +252,40 @@ fn bundle_exports() {
   assert_eq!(output.stderr, b"");
 }
 
+#[test]
+fn bundle_circular() {
+  use tempfile::TempDir;
+
+  // First we have to generate a bundle of some module that has exports.
+  let circular1 = util::root_path().join("cli/tests/subdir/circular1.ts");
+  assert!(circular1.is_file());
+  let t = TempDir::new().expect("tempdir fail");
+  let bundle = t.path().join("circular1.bundle.js");
+  let mut deno = util::deno_cmd()
+    .current_dir(util::root_path())
+    .arg("bundle")
+    .arg(circular1)
+    .arg(&bundle)
+    .spawn()
+    .expect("failed to spawn script");
+  let status = deno.wait().expect("failed to wait for the child process");
+  assert!(status.success());
+  assert!(bundle.is_file());
+
+  let output = util::deno_cmd()
+    .current_dir(util::root_path())
+    .arg("run")
+    .arg(&bundle)
+    .output()
+    .expect("failed to spawn script");
+  // check the output of the the bundle program.
+  assert!(std::str::from_utf8(&output.stdout)
+    .unwrap()
+    .trim()
+    .ends_with("f1\nf2"));
+  assert_eq!(output.stderr, b"");
+}
+
 // TODO(#2933): Rewrite this test in rust.
 #[test]
 fn repl_test() {

cli/tests/subdir/circular1.ts

@@ -0,0 +1,7 @@
+import * as circular2 from "./circular2.ts";
+
+export function f1(): void {
+  console.log("f1");
+}
+
+circular2.f2();

cli/tests/subdir/circular2.ts

@@ -0,0 +1,7 @@
+import * as circular1 from "./circular1.ts";
+
+export function f2(): void {
+  console.log("f2");
+}
+
+circular1.f1();

deno_typescript/lib.rs

@@ -245,7 +245,7 @@ pub fn get_asset(name: &str) -> Option<&'static str> {
     };
   }
   match name {
-    "bundle_loader.js" => Some(include_str!("bundle_loader.js")),
+    "system_loader.js" => Some(include_str!("system_loader.js")),
     "bootstrap.ts" => Some("console.log(\"hello deno\");"),
     "typescript.d.ts" => inc!("typescript.d.ts"),
     "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),

deno_typescript/system_loader.js

@@ -0,0 +1,85 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+// This is a specialised implementation of a System module loader.
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+let System;
+let __inst;
+
+(() => {
+  const mMap = new Map();
+  System = {
+    register(id, deps, f) {
+      mMap.set(id, {
+        id,
+        deps,
+        f,
+        exp: {}
+      });
+    }
+  };
+
+  const gC = (data, main) => {
+    const { id } = data;
+    return {
+      id,
+      import: async id => mMap.get(id)?.exp,
+      meta: { url: id, main }
+    };
+  };
+
+  const gE = data => {
+    const { exp } = data;
+    return (id, value) => {
+      const values = typeof id === "string" ? { [id]: value } : id;
+      for (const [id, value] of Object.entries(values)) {
+        Object.defineProperty(exp, id, {
+          value,
+          writable: true,
+          enumerable: true
+        });
+      }
+    };
+  };
+
+  const iQ = [];
+
+  const enq = ids => {
+    for (const id of ids) {
+      if (!iQ.includes(id)) {
+        const { deps } = mMap.get(id);
+        iQ.push(id);
+        enq(deps);
+      }
+    }
+  };
+
+  const dr = async main => {
+    const rQ = [];
+    let id;
+    while ((id = iQ.pop())) {
+      const m = mMap.get(id);
+      const { f } = m;
+      if (!f) {
+        return;
+      }
+      rQ.push([m.deps, f(gE(m), gC(m, id === main))]);
+      m.f = undefined;
+    }
+    let r;
+    while ((r = rQ.shift())) {
+      const [deps, { execute, setters }] = r;
+      for (let i = 0; i < deps.length; i++) setters[i](mMap.get(deps[i])?.exp);
+      const e = execute();
+      if (e) await e;
+    }
+  };
+
+  __inst = async id => {
+    System = undefined;
+    __inst = undefined;
+    enq([id]);
+    await dr(id);
+    return mMap.get(id)?.exp;
+  };
+})();