loader: support .wasm imports

Author: kevinkassimo

.prettierignore

@@ -1,3 +1,4 @@
+cli/compilers/wasm_wrap.js
 cli/tests/error_syntax.js
 cli/tests/badly_formatted.js
 cli/tests/top_level_for_await.js

Cargo.lock

@@ -27,6 +27,7 @@ deno_typescript = { path = "../deno_typescript", version = "0.23.0" }
 
 ansi_term = "0.12.1"
 atty = "0.2.13"
+base64 = "0.11.0"
 byteorder = "1.3.2"
 clap = "2.33.0"
 dirs = "2.0.2"

cli/Cargo.toml

@@ -5,10 +5,12 @@ use futures::Future;
 mod js;
 mod json;
 mod ts;
+mod wasm;
 
 pub use js::JsCompiler;
 pub use json::JsonCompiler;
 pub use ts::TsCompiler;
+pub use wasm::WasmCompiler;
 
 #[derive(Debug, Clone)]
 pub struct CompiledModule {

cli/compilers/mod.rs

@@ -0,0 +1,37 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use crate::compilers::CompiledModule;
+use crate::compilers::CompiledModuleFuture;
+use crate::file_fetcher::SourceFile;
+
+// TODO(kevinkassimo): This is a hack to encode/decode data as base64 string.
+// (Since Deno namespace might not be available, Deno.read may fail).
+// Binary data is already available through source_file.source_code.
+// If this is proven too wasteful in practice, refactor this.
+
+// Ref: https://webassembly.github.io/esm-integration/js-api/index.html#esm-integration
+
+// Only default exports is support atm.
+// Node.js supports named import since its dynamic module creation allows
+// running some code before transformation:
+// https://github.com/nodejs/node/blob/35ec01097b2a397ad0a22aac536fe07514876e21/lib/internal/modules/esm/translators.js#L190-L210
+// We need to expose worker to compilers to achieve that.
+
+pub struct WasmCompiler {}
+
+impl WasmCompiler {
+  pub fn compile_async(
+    self: &Self,
+    source_file: &SourceFile,
+  ) -> Box<CompiledModuleFuture> {
+    let code = wrap_wasm_code(&source_file.source_code);
+    let module = CompiledModule {
+      code,
+      name: source_file.url.to_string(),
+    };
+    Box::new(futures::future::ok(module))
+  }
+}
+
+pub fn wrap_wasm_code<T: ?Sized + AsRef<[u8]>>(input: &T) -> String {
+  format!(include_str!("./wasm_wrap.js"), base64::encode(input))
+}

cli/compilers/wasm.rs

@@ -0,0 +1,25 @@
+function base64ToUint8Array(data) {{
+  const binString = window.atob(data);
+  const size = binString.length;
+  const bytes = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {{
+    bytes[i] = binString.charCodeAt(i);
+  }}
+  return bytes;
+}}
+
+const buffer = base64ToUint8Array("{}");
+const compiled = await WebAssembly.compile(buffer);
+
+const imports = new Set(
+  WebAssembly.Module.imports(compiled).map(m => m.module)
+);
+
+const importObject = Object.create(null);
+for (const module of imports) {{
+  importObject[module] = await import(module);
+}}
+
+const instance = new WebAssembly.Instance(compiled, importObject);
+
+export default instance.exports;

cli/compilers/wasm_wrap.js

@@ -491,6 +491,7 @@ fn map_file_extension(path: &Path) -> msg::MediaType {
       Some("jsx") => msg::MediaType::JSX,
       Some("mjs") => msg::MediaType::JavaScript,
       Some("json") => msg::MediaType::Json,
+      Some("wasm") => msg::MediaType::Wasm,
       _ => msg::MediaType::Unknown,
     },
   }
@@ -1503,6 +1504,10 @@ mod tests {
       map_file_extension(Path::new("foo/bar.json")),
       msg::MediaType::Json
     );
+    assert_eq!(
+      map_file_extension(Path::new("foo/bar.wasm")),
+      msg::MediaType::Wasm
+    );
     assert_eq!(
       map_file_extension(Path::new("foo/bar.txt")),
       msg::MediaType::Unknown
@@ -1544,6 +1549,10 @@ mod tests {
       map_content_type(Path::new("foo/bar.json"), None),
       msg::MediaType::Json
     );
+    assert_eq!(
+      map_content_type(Path::new("foo/bar.wasm"), None),
+      msg::MediaType::Wasm
+    );
     assert_eq!(
       map_content_type(Path::new("foo/bar"), None),
       msg::MediaType::Unknown

cli/file_fetcher.rs

@@ -3,6 +3,7 @@ use crate::compilers::CompiledModule;
 use crate::compilers::JsCompiler;
 use crate::compilers::JsonCompiler;
 use crate::compilers::TsCompiler;
+use crate::compilers::WasmCompiler;
 use crate::deno_dir;
 use crate::deno_error::permission_denied;
 use crate::file_fetcher::SourceFileFetcher;
@@ -45,6 +46,7 @@ pub struct GlobalState {
   pub js_compiler: JsCompiler,
   pub json_compiler: JsonCompiler,
   pub ts_compiler: TsCompiler,
+  pub wasm_compiler: WasmCompiler,
   pub lockfile: Option<Mutex<Lockfile>>,
 }
 
@@ -111,6 +113,7 @@ impl ThreadSafeGlobalState {
       ts_compiler,
       js_compiler: JsCompiler {},
       json_compiler: JsonCompiler {},
+      wasm_compiler: WasmCompiler {},
       lockfile,
     };
 
@@ -130,6 +133,7 @@ impl ThreadSafeGlobalState {
       .and_then(move |out| match out.media_type {
         msg::MediaType::Unknown => state1.js_compiler.compile_async(&out),
         msg::MediaType::Json => state1.json_compiler.compile_async(&out),
+        msg::MediaType::Wasm => state1.wasm_compiler.compile_async(&out),
         msg::MediaType::TypeScript
         | msg::MediaType::TSX
         | msg::MediaType::JSX => {

cli/global_state.rs

@@ -28,9 +28,14 @@ enum MediaType {
   TypeScript = 2,
   TSX = 3,
   Json = 4,
-  Unknown = 5
+  Wasm = 5,
+  Unknown = 6
 }
 
+// ts.Extension does not contain Wasm type.
+// Forcefully create a marker of such type instead.
+const WASM_MARKER = (-1 as unknown) as ts.Extension;
+
 // Startup boilerplate. This is necessary because the compiler has its own
 // snapshot. (It would be great if we could remove these things or centralize
 // them somewhere else.)
@@ -145,13 +150,17 @@ class SourceFile {
   sourceCode!: string;
   tsSourceFile?: ts.SourceFile;
   url!: string;
+  isWasm = false;
 
   constructor(json: SourceFileJson) {
     if (SourceFile._moduleCache.has(json.url)) {
       throw new TypeError("SourceFile already exists");
     }
     Object.assign(this, json);
     this.extension = getExtension(this.url, this.mediaType);
+    if (this.extension === WASM_MARKER) {
+      this.isWasm = true;
+    }
     SourceFile._moduleCache.set(this.url, this);
   }
 
@@ -280,6 +289,19 @@ async function processImports(
   assert(sourceFiles.length === specifiers.length);
   for (let i = 0; i < sourceFiles.length; i++) {
     const sourceFileJson = sourceFiles[i];
+    if (sourceFileJson.mediaType === MediaType.Wasm) {
+      util.log(
+        "compiler::processImports: WASM import",
+        sourceFileJson.filename
+      );
+      // Create declaration file on the fly.
+      const _ = new SourceFile({
+        filename: `${sourceFileJson.filename}.d.ts`,
+        url: `${sourceFileJson.url}.d.ts`,
+        mediaType: MediaType.TypeScript,
+        sourceCode: "export default any;"
+      });
+    }
     const sourceFile =
       SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
     sourceFile.cache(specifiers[i][0], referrer);
@@ -339,6 +361,9 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
       return ts.Extension.Tsx;
     case MediaType.Json:
       return ts.Extension.Json;
+    case MediaType.Wasm:
+      // Custom marker for Wasm type.
+      return WASM_MARKER;
     case MediaType.Unknown:
     default:
       throw TypeError("Cannot resolve extension.");
@@ -510,6 +535,14 @@ class Host implements ts.CompilerHost {
       if (!sourceFile) {
         return undefined;
       }
+      if (sourceFile.isWasm) {
+        // WASM import, use custom .d.ts declaration instead.
+        return {
+          resolvedFileName: `${sourceFile.url}.d.ts`,
+          isExternalLibraryImport: false,
+          extension: ts.Extension.Dts
+        };
+      }
       return {
         resolvedFileName: sourceFile.url,
         isExternalLibraryImport: specifier.startsWith(ASSETS),

cli/js/compiler.ts

@@ -74,7 +74,8 @@ pub enum MediaType {
   TypeScript = 2,
   TSX = 3,
   Json = 4,
-  Unknown = 5,
+  Wasm = 5,
+  Unknown = 6,
 }
 
 pub fn enum_name_media_type(mt: MediaType) -> &'static str {
@@ -84,6 +85,7 @@ pub fn enum_name_media_type(mt: MediaType) -> &'static str {
     MediaType::TypeScript => "TypeScript",
     MediaType::TSX => "TSX",
     MediaType::Json => "Json",
+    MediaType::Wasm => "Wasm",
     MediaType::Unknown => "Unknown",
   }
 }

cli/msg.rs

@@ -0,0 +1,24 @@
+import wasmExports from "./051_wasm_import/simple.wasm";
+import { state } from "./051_wasm_import/wasm-dep.js";
+
+const { add, addImported, addRemote } = wasmExports;
+
+function assertEquals(actual: unknown, expected: unknown, msg?: string): void {
+  if (actual !== expected) {
+    throw new Error(msg);
+  }
+}
+
+assertEquals(state, "WASM Start Executed", "Incorrect state");
+
+assertEquals(add(10, 20), 30, "Incorrect add");
+
+assertEquals(addImported(0), 42, "Incorrect addImported");
+
+assertEquals(state, "WASM JS Function Executed", "Incorrect state");
+
+assertEquals(addImported(1), 43, "Incorrect addImported");
+
+assertEquals(addRemote(1), 2020, "Incorrect addRemote");
+
+console.log("Passed");

cli/tests/051_wasm_import.ts

@@ -0,0 +1 @@
+Passed

cli/tests/051_wasm_import.ts.out

@@ -0,0 +1,3 @@
+export function jsRemoteFn(): number {
+  return 2019;
+}

cli/tests/051_wasm_import/remote.ts

@@ -0,0 +1,31 @@
+;; From https://github.com/nodejs/node/blob/bbc254db5db672643aad89a436a4938412a5704e/test/fixtures/es-modules/simple.wat
+;; MIT Licensed
+;; $ wat2wasm simple.wat -o simple.wasm
+
+(module
+  (import "./wasm-dep.js" "jsFn" (func $jsFn (result i32)))
+  (import "./wasm-dep.js" "jsInitFn" (func $jsInitFn))
+  (import "http://localhost:4545/cli/tests/051_wasm_import/remote.ts" "jsRemoteFn" (func $jsRemoteFn (result i32)))
+  (export "add" (func $add))
+  (export "addImported" (func $addImported))
+  (export "addRemote" (func $addRemote))
+  (start $startFn)
+  (func $startFn
+    call $jsInitFn
+  )
+  (func $add (param $a i32) (param $b i32) (result i32)
+    local.get $a
+    local.get $b
+    i32.add
+  )
+  (func $addImported (param $a i32) (result i32)
+    local.get $a
+    call $jsFn
+    i32.add
+  )
+  (func $addRemote (param $a i32) (result i32)
+    local.get $a
+    call $jsRemoteFn
+    i32.add
+  )
+)

cli/tests/051_wasm_import/simple.wasm

@@ -0,0 +1,17 @@
+function assertEquals(actual, expected, msg) {
+  if (actual !== expected) {
+    throw new Error(msg || "");
+  }
+}
+
+export function jsFn() {
+  state = "WASM JS Function Executed";
+  return 42;
+}
+
+export let state = "JS Function Executed";
+
+export function jsInitFn() {
+  assertEquals(state, "JS Function Executed", "Incorrect state");
+  state = "WASM Start Executed";
+}

cli/tests/051_wasm_import/simple.wat

@@ -356,6 +356,11 @@ itest!(_050_more_jsons {
   output: "050_more_jsons.ts.out",
 });
 
+itest!(_051_wasm_import {
+  args: "run --reload --allow-net --allow-read 051_wasm_import.ts",
+  output: "051_wasm_import.ts.out",
+});
+
 itest!(lock_check_ok {
   args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts",
   output: "003_relative_import.ts.out",