Support loading external TS lib files

Fixes denoland#3726

This PR provides support for loading TypeScript libs that are not
bundled with Deno lazily.  It also provides the ability for Deno to
support the triple-slash lib reference.  There are a couple other minor
fixes and improvements as well.

Author: kitsonk

cli/build.rs

@@ -14,13 +14,12 @@ fn op_fetch_asset(
 ) -> impl Fn(&[u8], Option<ZeroCopyBuf>) -> CoreOp {
   move |control: &[u8], zero_copy_buf: Option<ZeroCopyBuf>| -> CoreOp {
     assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in this op.
-    let custom_assets = custom_assets.clone();
     let name = std::str::from_utf8(control).unwrap();
 
     let asset_code = if let Some(source_code) = deno_typescript::get_asset(name)
     {
       source_code.to_string()
-    } else if let Some(asset_path) = custom_assets.get(name) {
+    } else if let Some(asset_path) = custom_assets.clone().get(name) {
       let source_code_vec =
         std::fs::read(&asset_path).expect("Asset not found");
       let source_code = std::str::from_utf8(&source_code_vec).unwrap();

cli/file_fetcher.rs

@@ -7,6 +7,7 @@ use crate::http_util;
 use crate::http_util::create_http_client;
 use crate::http_util::FetchOnceResult;
 use crate::http_util::ResultPayload;
+use crate::js::get_remote_lib_str;
 use crate::msg;
 use crate::progress::Progress;
 use deno_core::ErrBox;
@@ -161,13 +162,25 @@ impl SourceFileFetcher {
     futures::executor::block_on(fut).ok()
   }
 
+  /// Given a `name`, resolve a fully qualified URL to fetch the asset from and
+  /// return a future which resolves with the source file.
+  pub fn fetch_remote_asset_async(
+    &self,
+    name: &str,
+  ) -> Pin<Box<SourceFileFuture>> {
+    debug!("fetch_remote_asset_async name: {} ", name);
+    let specifier =
+      ModuleSpecifier::resolve_url(&get_remote_lib_str(name)).unwrap();
+    self.fetch_source_file_async(&specifier, None)
+  }
+
   pub fn fetch_source_file_async(
     &self,
     specifier: &ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
   ) -> Pin<Box<SourceFileFuture>> {
     let module_url = specifier.as_url().to_owned();
-    debug!("fetch_source_file. specifier {} ", &module_url);
+    debug!("fetch_source_file_async specifier: {} ", &module_url);
 
     // Check if this file was already fetched and can be retrieved from in-process cache.
     if let Some(source_file) = self.source_file_cache.get(specifier.to_string())
@@ -368,18 +381,13 @@ impl SourceFileFetcher {
       }
       Ok(c) => c,
     };
-    let media_type = map_content_type(
-      &filepath,
-      source_code_headers.mime_type.as_ref().map(String::as_str),
-    );
+    let media_type =
+      map_content_type(&filepath, source_code_headers.mime_type.as_deref());
     let types_url = match media_type {
       msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
         &module_url,
         &source_code,
-        source_code_headers
-          .x_typescript_types
-          .as_ref()
-          .map(String::as_str),
+        source_code_headers.x_typescript_types.as_deref(),
       ),
       _ => None,
     };
@@ -517,16 +525,14 @@ impl SourceFileFetcher {
             .location
             .join(dir.deps_cache.get_cache_filename(&module_url));
 
-          let media_type = map_content_type(
-            &filepath,
-            maybe_content_type.as_ref().map(String::as_str),
-          );
+          let media_type =
+            map_content_type(&filepath, maybe_content_type.as_deref());
 
           let types_url = match media_type {
             msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
               &module_url,
               &source,
-              x_typescript_types.as_ref().map(String::as_str),
+              x_typescript_types.as_deref(),
             ),
             _ => None,
           };
@@ -1685,6 +1691,17 @@ mod tests {
     }
   }
 
+  #[test]
+  fn test_fetch_remote_asset() {
+    let (_temp_dir, fetcher) = test_setup();
+
+    tokio_util::run(fetcher.fetch_remote_asset_async("lib.dom.d.ts").map(
+      |r| {
+        assert!(r.is_ok());
+      },
+    ));
+  }
+
   #[test]
   fn test_map_file_extension() {
     assert_eq!(

cli/js.rs

@@ -1,5 +1,15 @@
 pub const TS_VERSION: &str = env!("TS_VERSION");
 
+pub fn get_remote_lib_str(name: &str) -> String {
+  format!(
+    "{}{}{}{}",
+    "https://raw.githubusercontent.com/microsoft/TypeScript/v",
+    env!("TS_VERSION"),
+    "/lib/",
+    name
+  )
+}
+
 pub static CLI_SNAPSHOT: &[u8] =
   include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
 pub static CLI_SNAPSHOT_MAP: &[u8] =
@@ -54,3 +64,14 @@ fn compiler_snapshot() {
     "#,
   ));
 }
+
+#[test]
+fn test_get_remote_url() {
+  let actual = get_remote_lib_str("lib.dom.d.ts");
+  let expected = concat!(
+    "https://raw.githubusercontent.com/microsoft/TypeScript/v",
+    env!("TS_VERSION"),
+    "/lib/lib.dom.d.ts"
+  );
+  assert_eq!(expected, actual);
+}

cli/js/compiler.ts

@@ -10,6 +10,8 @@
 // Either of these functions must be called when creating isolate
 // to properly setup runtime.
 
+// <reference types="../../deno_typescript/globals" />
+
 // NOTE: this import has side effects!
 import "./ts_global.d.ts";
 

cli/js/compiler_api.ts

@@ -101,6 +101,10 @@ export interface CompilerOptions {
    * Does not apply to `"esnext"` target. */
   useDefineForClassFields?: boolean;
 
+  /** List of library files to be included in the compilation.  If omitted,
+   * then the Deno main runtime libs are used. */
+  lib?: string[];
+
   /** The locale to use to show error messages. */
   locale?: string;
 

cli/js/compiler_bootstrap.ts

@@ -1,6 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { ASSETS, CompilerHostTarget, Host } from "./compiler_host.ts";
+import { CompilerHostTarget, Host } from "./compiler_host.ts";
+import { ASSETS } from "./compiler_sourcefile.ts";
 import { getAsset } from "./compiler_util.ts";
 
 // NOTE: target doesn't really matter here,
@@ -14,18 +15,11 @@ const options = host.getCompilationSettings();
 
 // This is a hacky way of adding our libs to the libs available in TypeScript()
 // as these are internal APIs of TypeScript which maintain valid libs
-/* eslint-disable @typescript-eslint/no-explicit-any */
-(ts as any).libs.push(
-  "deno_ns",
-  "deno_window",
-  "deno_worker",
-  "deno_shared_globals"
-);
-(ts as any).libMap.set("deno_ns", "lib.deno.ns.d.ts");
-(ts as any).libMap.set("deno_window", "lib.deno.window.d.ts");
-(ts as any).libMap.set("deno_worker", "lib.deno.worker.d.ts");
-(ts as any).libMap.set("deno_shared_globals", "lib.deno.shared_globals.d.ts");
-/* eslint-enable @typescript-eslint/no-explicit-any */
+ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals");
+ts.libMap.set("deno.ns", "lib.deno.ns.d.ts");
+ts.libMap.set("deno.window", "lib.deno.window.d.ts");
+ts.libMap.set("deno.worker", "lib.deno.worker.d.ts");
+ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts");
 
 // this pre-populates the cache at snapshot time of our library files, so they
 // are available in the future when needed.

cli/js/compiler_host.ts

@@ -1,6 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { MediaType, SourceFile } from "./compiler_sourcefile.ts";
+import { ASSETS, MediaType, SourceFile } from "./compiler_sourcefile.ts";
 import { OUT_DIR, WriteFileCallback, getAsset } from "./compiler_util.ts";
 import { cwd } from "./dir.ts";
 import { assert, notImplemented } from "./util.ts";
@@ -18,8 +18,14 @@ export enum CompilerHostTarget {
 }
 
 export interface CompilerHostOptions {
+  /** Flag determines if the host should assume a single bundle output. */
   bundle?: boolean;
+
+  /** Determines what the default library that should be used when type checking
+   * TS code. */
   target: CompilerHostTarget;
+
+  /** A function to be used when the program emit occurs to write out files. */
   writeFile: WriteFileCallback;
 }
 
@@ -28,8 +34,6 @@ export interface ConfigureResponse {
   diagnostics?: ts.Diagnostic[];
 }
 
-export const ASSETS = "$asset$";
-
 /** Options that need to be used when generating a bundle (either trusted or
  * runtime). */
 export const defaultBundlerOptions: ts.CompilerOptions = {
@@ -96,7 +100,7 @@ const ignoredCompilerOptions: readonly string[] = [
   "inlineSources",
   "init",
   "isolatedModules",
-  "lib",
+  // "lib",
   "listEmittedFiles",
   "listFiles",
   "mapRoot",
@@ -141,7 +145,10 @@ export class Host implements ts.CompilerHost {
   private _writeFile: WriteFileCallback;
 
   private _getAsset(filename: string): SourceFile {
-    const url = filename.split("/").pop()!;
+    const lastSegment = filename.split("/").pop()!;
+    const url = ts.libMap.has(lastSegment)
+      ? ts.libMap.get(lastSegment)!
+      : lastSegment;
     const sourceFile = SourceFile.get(url);
     if (sourceFile) {
       return sourceFile;
@@ -150,7 +157,7 @@ export class Host implements ts.CompilerHost {
     const sourceCode = getAsset(name);
     return new SourceFile({
       url,
-      filename,
+      filename: `${ASSETS}/${name}`,
       mediaType: MediaType.TypeScript,
       sourceCode
     });
@@ -230,6 +237,7 @@ export class Host implements ts.CompilerHost {
   }
 
   getDefaultLibFileName(_options: ts.CompilerOptions): string {
+    util.log("compiler::host.getDefaultLibFileName()");
     switch (this._target) {
       case CompilerHostTarget.Main:
       case CompilerHostTarget.Runtime:
@@ -259,7 +267,7 @@ export class Host implements ts.CompilerHost {
       if (!sourceFile.tsSourceFile) {
         assert(sourceFile.sourceCode != null);
         sourceFile.tsSourceFile = ts.createSourceFile(
-          fileName,
+          fileName.startsWith(ASSETS) ? sourceFile.filename : fileName,
           sourceFile.sourceCode,
           languageVersion
         );

cli/js/compiler_sourcefile.ts

@@ -26,6 +26,8 @@ export interface SourceFileJson {
   sourceCode: string;
 }
 
+export const ASSETS = "$asset$";
+
 /** Returns the TypeScript Extension enum for a given media type. */
 function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
   switch (mediaType) {
@@ -102,7 +104,7 @@ export class SourceFile {
     this.processed = true;
     const files = (this.importedFiles = [] as Array<[string, string]>);
 
-    function process(references: ts.FileReference[]): void {
+    function process(references: Array<{ fileName: string }>): void {
       for (const { fileName } of references) {
         files.push([fileName, fileName]);
       }
@@ -126,7 +128,15 @@ export class SourceFile {
       process(importedFiles);
     }
     process(referencedFiles);
-    process(libReferenceDirectives);
+    // built in libs comes across as `"dom"` for example, and should be filtered
+    // out during pre-processing as they are either already cached or they will
+    // be lazily fetched by the compiler host.  Ones that contain full files are
+    // not filtered out and will be fetched as normal.
+    process(
+      libReferenceDirectives.filter(
+        ({ fileName }) => !ts.libMap.has(fileName.toLowerCase())
+      )
+    );
     process(typeReferenceDirectives);
     return files;
   }

cli/js/compiler_util.ts

@@ -92,28 +92,39 @@ function cache(
 }
 
 let OP_FETCH_ASSET: number;
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
 
-/**
- * This op is called only during snapshotting.
- *
- * We really don't want to depend on JSON dispatch
- * during snapshotting, so this op exchanges strings with Rust
- * as raw byte arrays.
- */
+/** Retrieve an asset from Rust. */
 export function getAsset(name: string): string {
+  // this path should only be called for assets that are lazily loaded at
+  // runtime
+  if (dispatch.OP_FETCH_REMOTE_ASSET) {
+    util.log("compiler_util::getAsset", name);
+    return sendSync(dispatch.OP_FETCH_REMOTE_ASSET, { name }).sourceCode;
+  }
+
+  // this path should only be taken during snapshotting
   if (!OP_FETCH_ASSET) {
     const ops = core.ops();
     const opFetchAsset = ops["fetch_asset"];
     assert(opFetchAsset, "OP_FETCH_ASSET is not registered");
     OP_FETCH_ASSET = opFetchAsset;
   }
 
-  const encoder = new TextEncoder();
-  const decoder = new TextDecoder();
+  // We really don't want to depend on JSON dispatch during snapshotting, so
+  // this op exchanges strings with Rust as raw byte arrays.
   const sourceCodeBytes = core.dispatch(OP_FETCH_ASSET, encoder.encode(name));
   return decoder.decode(sourceCodeBytes!);
 }
 
+/** Some assets are only lazily requested at runtime via the compiler APIs,
+ * This will invoke an op to request that.
+ */
+export function getRemoteAsset(name: string): { sourceCode: string } {
+  return sendSync(dispatch.OP_FETCH_REMOTE_ASSET, { name });
+}
+
 /** Generates a `writeFile` function which can be passed to the compiler `Host`
  * to use when emitting files. */
 export function createWriteFile(state: WriteFileState): WriteFileCallback {

cli/js/dispatch.ts

@@ -19,6 +19,7 @@ export let OP_APPLY_SOURCE_MAP: number;
 export let OP_FORMAT_ERROR: number;
 export let OP_CACHE: number;
 export let OP_RESOLVE_MODULES: number;
+export let OP_FETCH_REMOTE_ASSET: number;
 export let OP_FETCH_SOURCE_FILES: number;
 export let OP_OPEN: number;
 export let OP_CLOSE: number;
@@ -78,10 +79,6 @@ export let OP_SIGNAL_BIND: number;
 export let OP_SIGNAL_UNBIND: number;
 export let OP_SIGNAL_POLL: number;
 
-/** **WARNING:** This is only available during the snapshotting process and is
- * unavailable at runtime. */
-export let OP_FETCH_ASSET: number;
-
 const PLUGIN_ASYNC_HANDLER_MAP: Map<number, AsyncHandler> = new Map();
 
 export function setPluginAsyncHandler(

cli/js/lib.deno.ns.d.ts

@@ -1,7 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
 /// <reference no-default-lib="true" />
-/// <reference lib="esnext" />
 
 declare namespace Deno {
   /** The current process id of the runtime. */
@@ -1863,6 +1862,10 @@ declare namespace Deno {
      * Does not apply to `"esnext"` target. */
     useDefineForClassFields?: boolean;
 
+    /** List of library files to be included in the compilation.  If omitted,
+     * then the Deno main runtime libs are used. */
+    lib?: string[];
+
     /** The locale to use to show error messages. */
     locale?: string;