Make bundles fully standalone (#3325)

- Bundles are fully standalone. They now include the shared loader with
  `deno_typescript`.
- Refactor of the loader in `deno_typescript` to perform module
  instantiation in a more
- Change of behaviour when an output file is not specified on the CLI.
  Previously a default name was determined and the bundle written to that
  file, now the bundle will be sent to `stdout`.
- Refactors in the TypeScript compiler to be able to support the concept
  of a request type.  This provides a cleaner abstraction and makes it
  easier to support things like single module transpiles to the userland.
- Remove a "dangerous" circular dependency between `os.ts` and `deno.ts`,
  and define `pid` and `noColor` in a better way.
- Don't bind early to `console` in `repl.ts`.
- Add an integration test for generating a bundle.

Author: kitsonk

cli/compilers/ts.rs

@@ -156,20 +156,23 @@ impl CompiledFileMetadata {
 }
 /// Creates the JSON message send to compiler.ts's onmessage.
 fn req(
+  request_type: msg::CompilerRequestType,
   root_names: Vec<String>,
   compiler_config: CompilerConfig,
-  bundle: Option<String>,
+  out_file: Option<String>,
 ) -> Buf {
   let j = match (compiler_config.path, compiler_config.content) {
     (Some(config_path), Some(config_data)) => json!({
+      "type": request_type as i32,
       "rootNames": root_names,
-      "bundle": bundle,
+      "outFile": out_file,
       "configPath": config_path,
       "config": str::from_utf8(&config_data).unwrap(),
     }),
     _ => json!({
+      "type": request_type as i32,
       "rootNames": root_names,
-      "bundle": bundle,
+      "outFile": out_file,
     }),
   };
 
@@ -250,15 +253,20 @@ impl TsCompiler {
     self: &Self,
     global_state: ThreadSafeGlobalState,
     module_name: String,
-    out_file: String,
+    out_file: Option<String>,
   ) -> impl Future<Item = (), Error = ErrBox> {
     debug!(
       "Invoking the compiler to bundle. module_name: {}",
       module_name
     );
 
     let root_names = vec![module_name.clone()];
-    let req_msg = req(root_names, self.config.clone(), Some(out_file));
+    let req_msg = req(
+      msg::CompilerRequestType::Bundle,
+      root_names,
+      self.config.clone(),
+      out_file,
+    );
 
     let worker = TsCompiler::setup_worker(global_state.clone());
     let worker_ = worker.clone();
@@ -360,7 +368,12 @@ impl TsCompiler {
     );
 
     let root_names = vec![module_url.to_string()];
-    let req_msg = req(root_names, self.config.clone(), None);
+    let req_msg = req(
+      msg::CompilerRequestType::Compile,
+      root_names,
+      self.config.clone(),
+      None,
+    );
 
     let worker = TsCompiler::setup_worker(global_state.clone());
     let worker_ = worker.clone();
@@ -709,7 +722,7 @@ mod tests {
         .bundle_async(
           state.clone(),
           module_name,
-          String::from("$deno$/bundle.js"),
+          Some(String::from("$deno$/bundle.js")),
         )
         .then(|result| {
           assert!(result.is_ok());

cli/flags.rs

@@ -6,7 +6,6 @@ use clap::Arg;
 use clap::ArgMatches;
 use clap::Shell;
 use clap::SubCommand;
-use deno::ModuleSpecifier;
 use log::Level;
 use std;
 use std::str;
@@ -259,11 +258,16 @@ compiler.",
       SubCommand::with_name("bundle")
         .about("Bundle module and dependencies into single file")
         .long_about(
-          "Output a single JavaScript file with all dependencies
+          "Output a single JavaScript file with all dependencies.
+
+If a out_file argument is omitted, the output of the bundle will be sent to
+standard out.
 
 Example:
 
-  deno bundle https://deno.land/std/examples/colors.ts"
+  deno bundle https://deno.land/std/examples/colors.ts
+  
+  deno bundle https://deno.land/std/examples/colors.ts colors.bundle.js"
         )
           .arg(Arg::with_name("source_file").takes_value(true).required(true))
           .arg(Arg::with_name("out_file").takes_value(true).required(false)),
@@ -793,32 +797,6 @@ pub enum DenoSubcommand {
   Version,
 }
 
-fn get_default_bundle_filename(source_file: &str) -> String {
-  let specifier = ModuleSpecifier::resolve_url_or_path(source_file).unwrap();
-  let path_segments = specifier.as_url().path_segments().unwrap();
-  let file_name = path_segments.filter(|s| !s.is_empty()).last().unwrap();
-  let file_stem = file_name.trim_end_matches(".ts").trim_end_matches(".js");
-  format!("{}.bundle.js", file_stem)
-}
-
-#[test]
-fn test_get_default_bundle_filename() {
-  assert_eq!(get_default_bundle_filename("blah.ts"), "blah.bundle.js");
-  assert_eq!(
-    get_default_bundle_filename("http://example.com/blah.ts"),
-    "blah.bundle.js"
-  );
-  assert_eq!(get_default_bundle_filename("blah.js"), "blah.bundle.js");
-  assert_eq!(
-    get_default_bundle_filename("http://example.com/blah.js"),
-    "blah.bundle.js"
-  );
-  assert_eq!(
-    get_default_bundle_filename("http://zombo.com/stuff/"),
-    "stuff.bundle.js"
-  );
-}
-
 pub fn flags_from_vec(
   args: Vec<String>,
 ) -> (DenoFlags, DenoSubcommand, Vec<String>) {
@@ -835,11 +813,13 @@ pub fn flags_from_vec(
     ("bundle", Some(bundle_match)) => {
       flags.allow_write = true;
       let source_file: &str = bundle_match.value_of("source_file").unwrap();
-      let out_file = bundle_match
-        .value_of("out_file")
-        .map(String::from)
-        .unwrap_or_else(|| get_default_bundle_filename(source_file));
-      argv.extend(vec![source_file.to_string(), out_file.to_string()]);
+      let out_file = bundle_match.value_of("out_file").map(String::from);
+      match out_file {
+        Some(out_file) => {
+          argv.extend(vec![source_file.to_string(), out_file.to_string()])
+        }
+        _ => argv.extend(vec![source_file.to_string()]),
+      }
       DenoSubcommand::Bundle
     }
     ("completions", Some(completions_match)) => {

cli/js/compiler.ts

@@ -31,6 +31,13 @@ enum MediaType {
   Unknown = 5
 }
 
+// Warning! The values in this enum are duplicated in cli/msg.rs
+// Update carefully!
+enum CompilerRequestType {
+  Compile = 0,
+  Bundle = 1
+}
+
 // Startup boilerplate. This is necessary because the compiler has its own
 // snapshot. (It would be great if we could remove these things or centralize
 // them somewhere else.)
@@ -44,16 +51,23 @@ window["denoMain"] = denoMain;
 
 const ASSETS = "$asset$";
 const OUT_DIR = "$deno$";
+const BUNDLE_LOADER = "bundle_loader.js";
 
 /** The format of the work message payload coming from the privileged side */
-interface CompilerReq {
+type CompilerRequest = {
   rootNames: string[];
-  bundle?: string;
   // TODO(ry) add compiler config to this interface.
   // options: ts.CompilerOptions;
   configPath?: string;
   config?: string;
-}
+} & (
+  | {
+      type: CompilerRequestType.Compile;
+    }
+  | {
+      type: CompilerRequestType.Bundle;
+      outFile?: string;
+    });
 
 interface ConfigureResponse {
   ignoredOptions?: string[];
@@ -271,7 +285,7 @@ function fetchSourceFiles(
 async function processImports(
   specifiers: Array<[string, string]>,
   referrer = ""
-): Promise<void> {
+): Promise<SourceFileJson[]> {
   if (!specifiers.length) {
     return;
   }
@@ -287,6 +301,7 @@ async function processImports(
       await processImports(sourceFile.imports(), sourceFile.url);
     }
   }
+  return sourceFiles;
 }
 
 /** Utility function to turn the number of bytes into a human readable
@@ -314,16 +329,36 @@ function cache(extension: string, moduleId: string, contents: string): void {
 const encoder = new TextEncoder();
 
 /** Given a fileName and the data, emit the file to the file system. */
-function emitBundle(fileName: string, data: string): void {
+function emitBundle(
+  rootNames: string[],
+  fileName: string | undefined,
+  data: string,
+  sourceFiles: readonly ts.SourceFile[]
+): void {
   // For internal purposes, when trying to emit to `$deno$` just no-op
-  if (fileName.startsWith("$deno$")) {
+  if (fileName && fileName.startsWith("$deno$")) {
     console.warn("skipping emitBundle", fileName);
     return;
   }
-  const encodedData = encoder.encode(data);
-  console.log(`Emitting bundle to "${fileName}"`);
-  writeFileSync(fileName, encodedData);
-  console.log(`${humanFileSize(encodedData.length)} emitted.`);
+  const loader = fetchAsset(BUNDLE_LOADER);
+  // when outputting to AMD and a single outfile, TypeScript makes up the module
+  // specifiers which are used to define the modules, and doesn't expose them
+  // publicly, so we have to try to replicate
+  const sources = sourceFiles.map(sf => sf.fileName);
+  const sharedPath = util.commonPath(sources);
+  rootNames = rootNames.map(id =>
+    id.replace(sharedPath, "").replace(/\.\w+$/i, "")
+  );
+  const instantiate = `instantiate(${JSON.stringify(rootNames)});\n`;
+  const bundle = `${loader}\n${data}\n${instantiate}`;
+  if (fileName) {
+    const encodedData = encoder.encode(bundle);
+    console.warn(`Emitting bundle to "${fileName}"`);
+    writeFileSync(fileName, encodedData);
+    console.warn(`${humanFileSize(encodedData.length)} emitted.`);
+  } else {
+    console.log(bundle);
+  }
 }
 
 /** Returns the TypeScript Extension enum for a given media type. */
@@ -380,17 +415,23 @@ class Host implements ts.CompilerHost {
 
   /** Provides the `ts.HostCompiler` interface for Deno.
    *
+   * @param _rootNames A set of modules that are the ones that should be
+   *   instantiated first.  Used when generating a bundle.
    * @param _bundle Set to a string value to configure the host to write out a
    *   bundle instead of caching individual files.
    */
-  constructor(private _bundle?: string) {
-    if (this._bundle) {
+  constructor(
+    private _requestType: CompilerRequestType,
+    private _rootNames: string[],
+    private _outFile?: string
+  ) {
+    if (this._requestType === CompilerRequestType.Bundle) {
       // options we need to change when we are generating a bundle
       const bundlerOptions: ts.CompilerOptions = {
         module: ts.ModuleKind.AMD,
-        inlineSourceMap: true,
         outDir: undefined,
         outFile: `${OUT_DIR}/bundle.js`,
+        // disabled until we have effective way to modify source maps
         sourceMap: false
       };
       Object.assign(this._options, bundlerOptions);
@@ -531,10 +572,11 @@ class Host implements ts.CompilerHost {
   ): void {
     util.log("compiler::host.writeFile", fileName);
     try {
-      if (this._bundle) {
-        emitBundle(this._bundle, data);
+      assert(sourceFiles != null);
+      if (this._requestType === CompilerRequestType.Bundle) {
+        emitBundle(this._rootNames, this._outFile, data, sourceFiles!);
       } else {
-        assert(sourceFiles != null && sourceFiles.length == 1);
+        assert(sourceFiles.length == 1);
         const url = sourceFiles![0].fileName;
         const sourceFile = SourceFile.get(url);
 
@@ -579,16 +621,29 @@ class Host implements ts.CompilerHost {
 // lazy instantiating the compiler web worker
 window.compilerMain = function compilerMain(): void {
   // workerMain should have already been called since a compiler is a worker.
-  window.onmessage = async ({ data }: { data: CompilerReq }): Promise<void> => {
-    const { rootNames, configPath, config, bundle } = data;
-    util.log(">>> compile start", { rootNames, bundle });
+  window.onmessage = async ({
+    data: request
+  }: {
+    data: CompilerRequest;
+  }): Promise<void> => {
+    const { rootNames, configPath, config } = request;
+    util.log(">>> compile start", {
+      rootNames,
+      type: CompilerRequestType[request.type]
+    });
 
     // This will recursively analyse all the code for other imports, requesting
     // those from the privileged side, populating the in memory cache which
     // will be used by the host, before resolving.
-    await processImports(rootNames.map(rootName => [rootName, rootName]));
-
-    const host = new Host(bundle);
+    const resolvedRootModules = (await processImports(
+      rootNames.map(rootName => [rootName, rootName])
+    )).map(info => info.url);
+
+    const host = new Host(
+      request.type,
+      resolvedRootModules,
+      request.type === CompilerRequestType.Bundle ? request.outFile : undefined
+    );
     let emitSkipped = true;
     let diagnostics: ts.Diagnostic[] | undefined;
 
@@ -642,8 +697,9 @@ window.compilerMain = function compilerMain(): void {
 
       // We will only proceed with the emit if there are no diagnostics.
       if (diagnostics && diagnostics.length === 0) {
-        if (bundle) {
-          console.log(`Bundling "${bundle}"`);
+        if (request.type === CompilerRequestType.Bundle) {
+          // warning so it goes to stderr instead of stdout
+          console.warn(`Bundling "${resolvedRootModules.join(`", "`)}"`);
         }
         const emitResult = program.emit();
         emitSkipped = emitResult.emitSkipped;
@@ -662,7 +718,10 @@ window.compilerMain = function compilerMain(): void {
 
     postMessage(result);
 
-    util.log("<<< compile end", { rootNames, bundle });
+    util.log("<<< compile end", {
+      rootNames,
+      type: CompilerRequestType[request.type]
+    });
 
     // The compiler isolate exits after a single message.
     workerClose();

cli/js/deno.ts

@@ -112,9 +112,3 @@ export let pid: number;
 
 /** Reflects the NO_COLOR environment variable: https://no-color.org/ */
 export let noColor: boolean;
-
-// TODO(ry) This should not be exposed to Deno.
-export function _setGlobals(pid_: number, noColor_: boolean): void {
-  pid = pid_;
-  noColor = noColor_;
-}