Support loading additional TS lib files

Fixes #3726

This PR provides support for referencing other lib files that are not
used by default in Deno via the compiler APIs.

Author: kitsonk

cli/build.rs

@@ -14,13 +14,12 @@ fn op_fetch_asset(
 ) -> impl Fn(&[u8], Option<ZeroCopyBuf>) -> CoreOp {
   move |control: &[u8], zero_copy_buf: Option<ZeroCopyBuf>| -> CoreOp {
     assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in this op.
-    let custom_assets = custom_assets.clone();
     let name = std::str::from_utf8(control).unwrap();
 
     let asset_code = if let Some(source_code) = deno_typescript::get_asset(name)
     {
       source_code.to_string()
-    } else if let Some(asset_path) = custom_assets.get(name) {
+    } else if let Some(asset_path) = custom_assets.clone().get(name) {
       let source_code_vec =
         std::fs::read(&asset_path).expect("Asset not found");
       let source_code = std::str::from_utf8(&source_code_vec).unwrap();

cli/file_fetcher.rs

@@ -165,7 +165,7 @@ impl SourceFileFetcher {
     maybe_referrer: Option<ModuleSpecifier>,
   ) -> Pin<Box<SourceFileFuture>> {
     let module_url = specifier.as_url().to_owned();
-    debug!("fetch_source_file. specifier {} ", &module_url);
+    debug!("fetch_source_file_async specifier: {} ", &module_url);
 
     // Check if this file was already fetched and can be retrieved from in-process cache.
     if let Some(source_file) = self.source_file_cache.get(specifier.to_string())
@@ -368,18 +368,13 @@ impl SourceFileFetcher {
       }
       Ok(c) => c,
     };
-    let media_type = map_content_type(
-      &filepath,
-      source_code_headers.mime_type.as_ref().map(String::as_str),
-    );
+    let media_type =
+      map_content_type(&filepath, source_code_headers.mime_type.as_deref());
     let types_url = match media_type {
       msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
         &module_url,
         &source_code,
-        source_code_headers
-          .x_typescript_types
-          .as_ref()
-          .map(String::as_str),
+        source_code_headers.x_typescript_types.as_deref(),
       ),
       _ => None,
     };
@@ -515,17 +510,13 @@ impl SourceFileFetcher {
             .location
             .join(dir.deps_cache.get_cache_filename(&module_url));
 
-          let media_type = map_content_type(
-            &filepath,
-            maybe_content_type.as_ref().map(String::as_str),
-          );
+          let media_type =
+            map_content_type(&filepath, maybe_content_type.as_deref());
 
           let types_url = match media_type {
-            msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
-              &module_url,
-              &source,
-              x_typescript_types.as_ref().map(String::as_str),
-            ),
+            msg::MediaType::JavaScript | msg::MediaType::JSX => {
+              get_types_url(&module_url, &source, x_typescript_types.as_deref())
+            }
             _ => None,
           };
 

cli/js/compiler_api.ts

@@ -101,6 +101,10 @@ export interface CompilerOptions {
    * Does not apply to `"esnext"` target. */
   useDefineForClassFields?: boolean;
 
+  /** List of library files to be included in the compilation.  If omitted,
+   * then the Deno main runtime libs are used. */
+  lib?: string[];
+
   /** The locale to use to show error messages. */
   locale?: string;
 

cli/js/compiler_api_test.ts

@@ -46,6 +46,22 @@ test(async function compilerApiCompileOptions() {
   assert(actual["/foo.js"].startsWith("define("));
 });
 
+test(async function compilerApiCompileLib() {
+  const [diagnostics, actual] = await compile(
+    "/foo.ts",
+    {
+      "/foo.ts": `console.log(document.getElementById("foo"));
+        console.log(Deno.args);`
+    },
+    {
+      lib: ["dom", "es2018", "deno.ns"]
+    }
+  );
+  assert(diagnostics == null);
+  assert(actual);
+  assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]);
+});
+
 test(async function transpileOnlyApi() {
   const actual = await transpileOnly({
     "foo.ts": `export enum Foo { Foo, Bar, Baz };\n`

cli/js/compiler_bootstrap.ts

@@ -1,6 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { ASSETS, CompilerHostTarget, Host } from "./compiler_host.ts";
+import { CompilerHostTarget, Host } from "./compiler_host.ts";
+import { ASSETS } from "./compiler_sourcefile.ts";
 import { getAsset } from "./compiler_util.ts";
 
 // NOTE: target doesn't really matter here,
@@ -14,18 +15,11 @@ const options = host.getCompilationSettings();
 
 // This is a hacky way of adding our libs to the libs available in TypeScript()
 // as these are internal APIs of TypeScript which maintain valid libs
-/* eslint-disable @typescript-eslint/no-explicit-any */
-(ts as any).libs.push(
-  "deno_ns",
-  "deno_window",
-  "deno_worker",
-  "deno_shared_globals"
-);
-(ts as any).libMap.set("deno_ns", "lib.deno.ns.d.ts");
-(ts as any).libMap.set("deno_window", "lib.deno.window.d.ts");
-(ts as any).libMap.set("deno_worker", "lib.deno.worker.d.ts");
-(ts as any).libMap.set("deno_shared_globals", "lib.deno.shared_globals.d.ts");
-/* eslint-enable @typescript-eslint/no-explicit-any */
+ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals");
+ts.libMap.set("deno.ns", "lib.deno.ns.d.ts");
+ts.libMap.set("deno.window", "lib.deno.window.d.ts");
+ts.libMap.set("deno.worker", "lib.deno.worker.d.ts");
+ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts");
 
 // this pre-populates the cache at snapshot time of our library files, so they
 // are available in the future when needed.

cli/js/compiler_host.ts

@@ -1,6 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-import { MediaType, SourceFile } from "./compiler_sourcefile.ts";
+import { ASSETS, MediaType, SourceFile } from "./compiler_sourcefile.ts";
 import { OUT_DIR, WriteFileCallback, getAsset } from "./compiler_util.ts";
 import { cwd } from "./dir.ts";
 import { assert, notImplemented } from "./util.ts";
@@ -18,8 +18,14 @@ export enum CompilerHostTarget {
 }
 
 export interface CompilerHostOptions {
+  /** Flag determines if the host should assume a single bundle output. */
   bundle?: boolean;
+
+  /** Determines what the default library that should be used when type checking
+   * TS code. */
   target: CompilerHostTarget;
+
+  /** A function to be used when the program emit occurs to write out files. */
   writeFile: WriteFileCallback;
 }
 
@@ -28,8 +34,6 @@ export interface ConfigureResponse {
   diagnostics?: ts.Diagnostic[];
 }
 
-export const ASSETS = "$asset$";
-
 /** Options that need to be used when generating a bundle (either trusted or
  * runtime). */
 export const defaultBundlerOptions: ts.CompilerOptions = {
@@ -96,7 +100,6 @@ const ignoredCompilerOptions: readonly string[] = [
   "inlineSources",
   "init",
   "isolatedModules",
-  "lib",
   "listEmittedFiles",
   "listFiles",
   "mapRoot",
@@ -141,7 +144,10 @@ export class Host implements ts.CompilerHost {
   private _writeFile: WriteFileCallback;
 
   private _getAsset(filename: string): SourceFile {
-    const url = filename.split("/").pop()!;
+    const lastSegment = filename.split("/").pop()!;
+    const url = ts.libMap.has(lastSegment)
+      ? ts.libMap.get(lastSegment)!
+      : lastSegment;
     const sourceFile = SourceFile.get(url);
     if (sourceFile) {
       return sourceFile;
@@ -150,7 +156,7 @@ export class Host implements ts.CompilerHost {
     const sourceCode = getAsset(name);
     return new SourceFile({
       url,
-      filename,
+      filename: `${ASSETS}/${name}`,
       mediaType: MediaType.TypeScript,
       sourceCode
     });
@@ -230,6 +236,7 @@ export class Host implements ts.CompilerHost {
   }
 
   getDefaultLibFileName(_options: ts.CompilerOptions): string {
+    util.log("compiler::host.getDefaultLibFileName()");
     switch (this._target) {
       case CompilerHostTarget.Main:
       case CompilerHostTarget.Runtime:
@@ -259,7 +266,7 @@ export class Host implements ts.CompilerHost {
       if (!sourceFile.tsSourceFile) {
         assert(sourceFile.sourceCode != null);
         sourceFile.tsSourceFile = ts.createSourceFile(
-          fileName,
+          fileName.startsWith(ASSETS) ? sourceFile.filename : fileName,
           sourceFile.sourceCode,
           languageVersion
         );

cli/js/compiler_sourcefile.ts

@@ -26,6 +26,8 @@ export interface SourceFileJson {
   sourceCode: string;
 }
 
+export const ASSETS = "$asset$";
+
 /** Returns the TypeScript Extension enum for a given media type. */
 function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
   switch (mediaType) {
@@ -109,7 +111,7 @@ export class SourceFile {
     this.processed = true;
     const files = (this.importedFiles = [] as Array<[string, string]>);
 
-    function process(references: ts.FileReference[]): void {
+    function process(references: Array<{ fileName: string }>): void {
       for (const { fileName } of references) {
         files.push([fileName, fileName]);
       }
@@ -133,7 +135,15 @@ export class SourceFile {
       process(importedFiles);
     }
     process(referencedFiles);
-    process(libReferenceDirectives);
+    // built in libs comes across as `"dom"` for example, and should be filtered
+    // out during pre-processing as they are either already cached or they will
+    // be lazily fetched by the compiler host.  Ones that contain full files are
+    // not filtered out and will be fetched as normal.
+    process(
+      libReferenceDirectives.filter(
+        ({ fileName }) => !ts.libMap.has(fileName.toLowerCase())
+      )
+    );
     process(typeReferenceDirectives);
     return files;
   }

cli/js/compiler_util.ts

@@ -92,24 +92,28 @@ function cache(
 }
 
 let OP_FETCH_ASSET: number;
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
 
-/**
- * This op is called only during snapshotting.
- *
- * We really don't want to depend on JSON dispatch
- * during snapshotting, so this op exchanges strings with Rust
- * as raw byte arrays.
- */
+/** Retrieve an asset from Rust. */
 export function getAsset(name: string): string {
+  // this path should only be called for assets that are lazily loaded at
+  // runtime
+  if (dispatch.OP_FETCH_ASSET) {
+    util.log("compiler_util::getAsset", name);
+    return sendSync(dispatch.OP_FETCH_ASSET, { name }).sourceCode;
+  }
+
+  // this path should only be taken during snapshotting
   if (!OP_FETCH_ASSET) {
     const ops = core.ops();
     const opFetchAsset = ops["fetch_asset"];
     assert(opFetchAsset, "OP_FETCH_ASSET is not registered");
     OP_FETCH_ASSET = opFetchAsset;
   }
 
-  const encoder = new TextEncoder();
-  const decoder = new TextDecoder();
+  // We really don't want to depend on JSON dispatch during snapshotting, so
+  // this op exchanges strings with Rust as raw byte arrays.
   const sourceCodeBytes = core.dispatch(OP_FETCH_ASSET, encoder.encode(name));
   return decoder.decode(sourceCodeBytes!);
 }

cli/js/dispatch.ts

@@ -19,6 +19,7 @@ export let OP_APPLY_SOURCE_MAP: number;
 export let OP_FORMAT_ERROR: number;
 export let OP_CACHE: number;
 export let OP_RESOLVE_MODULES: number;
+export let OP_FETCH_ASSET: number;
 export let OP_FETCH_SOURCE_FILES: number;
 export let OP_OPEN: number;
 export let OP_CLOSE: number;
@@ -76,10 +77,6 @@ export let OP_SIGNAL_BIND: number;
 export let OP_SIGNAL_UNBIND: number;
 export let OP_SIGNAL_POLL: number;
 
-/** **WARNING:** This is only available during the snapshotting process and is
- * unavailable at runtime. */
-export let OP_FETCH_ASSET: number;
-
 const PLUGIN_ASYNC_HANDLER_MAP: Map<number, AsyncHandler> = new Map();
 
 export function setPluginAsyncHandler(

cli/js/lib.deno.ns.d.ts

@@ -248,7 +248,7 @@ declare namespace Deno {
   /** UNSTABLE: might move to Deno.symbols */
   export const EOF: unique symbol;
 
-  /** UNSTABLE: might move to Deno.symbols */
+  /** UNSTABLE: might move to Deno.symbols  */
   export type EOF = typeof EOF;
 
   /** UNSTABLE: maybe remove "SEEK_" prefix. Maybe capitalization wrong. */
@@ -1917,6 +1917,10 @@ declare namespace Deno {
      * Does not apply to `"esnext"` target. */
     useDefineForClassFields?: boolean;
 
+    /** List of library files to be included in the compilation.  If omitted,
+     * then the Deno main runtime libs are used. */
+    lib?: string[];
+
     /** The locale to use to show error messages. */
     locale?: string;
 

cli/js/lib.deno.shared_globals.d.ts

@@ -3,7 +3,10 @@
 /* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */
 
 /// <reference no-default-lib="true" />
-/// <reference lib="deno_ns" />
+// TODO: we need to remove this, but Fetch::Response::Body implements Reader
+// which requires Deno.EOF, and we shouldn't be leaking that, but https_proxy
+// at the least requires the Reader interface on Body, which it shouldn't
+/// <reference lib="deno.ns" />
 /// <reference lib="esnext" />
 
 // https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope

cli/js/lib.deno.window.d.ts

@@ -3,8 +3,8 @@
 /* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */
 
 /// <reference no-default-lib="true" />
-/// <reference lib="deno_ns" />
-/// <reference lib="deno_shared_globals" />
+/// <reference lib="deno.ns" />
+/// <reference lib="deno.shared_globals" />
 /// <reference lib="esnext" />
 
 declare interface Window extends WindowOrWorkerGlobalScope {

cli/js/lib.deno.worker.d.ts

@@ -3,7 +3,7 @@
 /* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */
 
 /// <reference no-default-lib="true" />
-/// <reference lib="deno_shared_globals" />
+/// <reference lib="deno.shared_globals" />
 /// <reference lib="esnext" />
 
 declare interface DedicatedWorkerGlobalScope extends WindowOrWorkerGlobalScope {

cli/js/ts_global.d.ts

@@ -16,4 +16,11 @@ declare global {
   namespace ts {
     export = ts_;
   }
+
+  namespace ts {
+    // this are marked @internal in TypeScript, but we need to access them,
+    // there is a risk these could change in future versions of TypeScript
+    export const libs: string[];
+    export const libMap: Map<string, string>;
+  }
 }