Support named exports

Author: kevinkassimo

.eslintignore

@@ -1,3 +1,4 @@
+cli/compilers/wasm_wrap.js
 cli/tests/error_syntax.js
 std/deno.d.ts
 std/prettier/vendor

cli/compilers/wasm.rs

@@ -2,36 +2,184 @@
 use crate::compilers::CompiledModule;
 use crate::compilers::CompiledModuleFuture;
 use crate::file_fetcher::SourceFile;
+use crate::global_state::ThreadSafeGlobalState;
+use crate::startup_data;
+use crate::state::*;
+use crate::worker::Worker;
+use deno::Buf;
+use futures::Future;
+use futures::IntoFuture;
+use serde_derive::Deserialize;
+use serde_json;
+use std::collections::HashMap;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, Mutex};
+use url::Url;
 
 // TODO(kevinkassimo): This is a hack to encode/decode data as base64 string.
 // (Since Deno namespace might not be available, Deno.read can fail).
 // Binary data is already available through source_file.source_code.
 // If this is proven too wasteful in practice, refactor this.
 
 // Ref: https://webassembly.github.io/esm-integration/js-api/index.html#esm-integration
-
-// Only default exports is support ATM.
-// Node.js supports named import since its dynamic module creation allows
-// running some code before transformation:
 // https://github.com/nodejs/node/blob/35ec01097b2a397ad0a22aac536fe07514876e21/lib/internal/modules/esm/translators.js#L190-L210
-// We need to expose worker to compilers to achieve that.
 
-pub struct WasmCompiler {}
+// Dynamically construct JS wrapper with custom static imports and named exports.
+// Boots up an internal worker to resolve imports/exports through query from V8.
+
+static WASM_WRAP: &str = include_str!("./wasm_wrap.js");
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct WasmModuleInfo {
+  import_list: Vec<String>,
+  export_list: Vec<String>,
+}
+
+pub struct WasmCompiler {
+  cache: Arc<Mutex<HashMap<Url, CompiledModule>>>,
+}
 
 impl WasmCompiler {
+  pub fn new() -> WasmCompiler {
+    Self {
+      cache: Arc::new(Mutex::new(HashMap::new())),
+    }
+  }
+
+  /// Create a new V8 worker with snapshot of WASM compiler and setup compiler's runtime.
+  fn setup_worker(global_state: ThreadSafeGlobalState) -> Worker {
+    let (int, ext) = ThreadSafeState::create_channels();
+    let worker_state =
+      ThreadSafeState::new(global_state.clone(), None, true, int)
+        .expect("Unable to create worker state");
+
+    // Count how many times we start the compiler worker.
+    global_state
+      .metrics
+      .compiler_starts
+      .fetch_add(1, Ordering::SeqCst);
+
+    let mut worker = Worker::new(
+      "WASM".to_string(),
+      startup_data::compiler_isolate_init(),
+      worker_state,
+      ext,
+    );
+    worker.execute("denoMain('WASM')").unwrap();
+    worker.execute("workerMain()").unwrap();
+    worker.execute("wasmCompilerMain()").unwrap();
+    worker
+  }
+
   pub fn compile_async(
     self: &Self,
+    global_state: ThreadSafeGlobalState,
     source_file: &SourceFile,
   ) -> Box<CompiledModuleFuture> {
-    let code = wrap_wasm_code(&source_file.source_code);
-    let module = CompiledModule {
-      code,
-      name: source_file.url.to_string(),
+    let cache = self.cache.clone();
+    let maybe_cached = {
+      cache
+        .lock()
+        .unwrap()
+        .get(&source_file.url)
+        .map(|c| c.clone())
     };
-    Box::new(futures::future::ok(module))
+    if let Some(m) = maybe_cached {
+      return Box::new(futures::future::ok(m.clone()));
+    }
+    let cache_ = self.cache.clone();
+
+    debug!(">>>>> wasm_compile_async START");
+    let base64_data = base64::encode(&source_file.source_code);
+    let worker = WasmCompiler::setup_worker(global_state.clone());
+    let worker_ = worker.clone();
+    let url = source_file.url.clone();
+
+    let fut = worker
+      .post_message(
+        serde_json::to_string(&base64_data)
+          .unwrap()
+          .into_boxed_str()
+          .into_boxed_bytes(),
+      )
+      .into_future()
+      .then(move |_| worker)
+      .then(move |result| {
+        if let Err(err) = result {
+          // TODO(ry) Need to forward the error instead of exiting.
+          eprintln!("{}", err.to_string());
+          std::process::exit(1);
+        }
+        debug!("Sent message to worker");
+        worker_.get_message()
+      })
+      .map_err(|_| panic!("not handled"))
+      .and_then(move |maybe_msg: Option<Buf>| {
+        debug!("Received message from worker");
+        let json_msg = maybe_msg.unwrap();
+        let module_info: WasmModuleInfo =
+          serde_json::from_slice(&json_msg).unwrap();
+        debug!("WASM module info: {:#?}", &module_info);
+        let code = wrap_wasm_code(
+          &base64_data,
+          &module_info.import_list,
+          &module_info.export_list,
+        );
+        debug!("Generated code: {}", &code);
+        let module = CompiledModule {
+          code,
+          name: url.to_string(),
+        };
+        {
+          cache_.lock().unwrap().insert(url.clone(), module.clone());
+        }
+        debug!("<<<<< wasm_compile_async END");
+        Ok(module)
+      });
+    Box::new(fut)
+  }
+}
+
+fn build_single_import(index: usize, origin: &str) -> String {
+  let origin_json = serde_json::to_string(origin).unwrap();
+  format!(
+    r#"import * as m{} from {};
+importObject[{}] = m{};
+"#,
+    index, &origin_json, &origin_json, index
+  )
+}
+
+fn build_imports(imports: &Vec<String>) -> String {
+  let mut code = String::from("");
+  for (index, origin) in imports.iter().enumerate() {
+    code.push_str(&build_single_import(index, origin));
+  }
+  code
+}
+
+fn build_single_export(name: &str) -> String {
+  format!("export const {} = instance.exports.{};\n", name, name)
+}
+
+fn build_exports(exports: &Vec<String>) -> String {
+  let mut code = String::from("");
+  for e in exports {
+    code.push_str(&build_single_export(e));
   }
+  code
 }
 
-pub fn wrap_wasm_code<T: ?Sized + AsRef<[u8]>>(input: &T) -> String {
-  format!(include_str!("./wasm_wrap.js"), base64::encode(input))
+fn wrap_wasm_code(
+  base64_data: &str,
+  imports: &Vec<String>,
+  exports: &Vec<String>,
+) -> String {
+  let imports_code = build_imports(imports);
+  let exports_code = build_exports(exports);
+  String::from(WASM_WRAP)
+    .replace("//IMPORTS\n", &imports_code)
+    .replace("//EXPORTS\n", &exports_code)
+    .replace("BASE64_DATA", base64_data)
 }

cli/compilers/wasm_wrap.js

@@ -1,25 +1,19 @@
-function base64ToUint8Array(data) {{
+const importObject = Object.create(null);
+//IMPORTS
+
+function base64ToUint8Array(data) {
   const binString = window.atob(data);
   const size = binString.length;
   const bytes = new Uint8Array(size);
-  for (let i = 0; i < size; i++) {{
+  for (let i = 0; i < size; i++) {
     bytes[i] = binString.charCodeAt(i);
-  }}
+  }
   return bytes;
-}}
+}
 
-const buffer = base64ToUint8Array("{}");
+const buffer = base64ToUint8Array("BASE64_DATA");
 const compiled = await WebAssembly.compile(buffer);
 
-const imports = new Set(
-  WebAssembly.Module.imports(compiled).map(m => m.module)
-);
-
-const importObject = Object.create(null);
-for (const module of imports) {{
-  importObject[module] = await import(module);
-}}
-
 const instance = new WebAssembly.Instance(compiled, importObject);
 
-export default instance.exports;
+//EXPORTS

cli/global_state.rs

@@ -113,7 +113,7 @@ impl ThreadSafeGlobalState {
       ts_compiler,
       js_compiler: JsCompiler {},
       json_compiler: JsonCompiler {},
-      wasm_compiler: WasmCompiler {},
+      wasm_compiler: WasmCompiler::new(),
       lockfile,
     };
 
@@ -133,7 +133,9 @@ impl ThreadSafeGlobalState {
       .and_then(move |out| match out.media_type {
         msg::MediaType::Unknown => state1.js_compiler.compile_async(&out),
         msg::MediaType::Json => state1.json_compiler.compile_async(&out),
-        msg::MediaType::Wasm => state1.wasm_compiler.compile_async(&out),
+        msg::MediaType::Wasm => {
+          state1.wasm_compiler.compile_async(state1.clone(), &out)
+        }
         msg::MediaType::TypeScript
         | msg::MediaType::TSX
         | msg::MediaType::JSX => {

cli/js/compiler.ts

@@ -32,10 +32,6 @@ enum MediaType {
   Unknown = 6
 }
 
-// ts.Extension does not contain Wasm type.
-// Forcefully create a marker of such type instead.
-const WASM_MARKER = (-1 as unknown) as ts.Extension;
-
 // Warning! The values in this enum are duplicated in cli/msg.rs
 // Update carefully!
 enum CompilerRequestType {
@@ -49,8 +45,8 @@ enum CompilerRequestType {
 const console = new Console(core.print);
 window.console = console;
 window.workerMain = workerMain;
-function denoMain(): void {
-  os.start(true, "TS");
+function denoMain(compilerType?: string): void {
+  os.start(true, compilerType || "TS");
 }
 window["denoMain"] = denoMain;
 
@@ -165,17 +161,13 @@ class SourceFile {
   sourceCode!: string;
   tsSourceFile?: ts.SourceFile;
   url!: string;
-  isWasm = false;
 
   constructor(json: SourceFileJson) {
     if (SourceFile._moduleCache.has(json.url)) {
       throw new TypeError("SourceFile already exists");
     }
     Object.assign(this, json);
     this.extension = getExtension(this.url, this.mediaType);
-    if (this.extension === WASM_MARKER) {
-      this.isWasm = true;
-    }
     SourceFile._moduleCache.set(this.url, this);
   }
 
@@ -300,19 +292,6 @@ async function processImports(
   assert(sourceFiles.length === specifiers.length);
   for (let i = 0; i < sourceFiles.length; i++) {
     const sourceFileJson = sourceFiles[i];
-    if (sourceFileJson.mediaType === MediaType.Wasm) {
-      util.log(
-        "compiler::processImports: WASM import",
-        sourceFileJson.filename
-      );
-      // Create declaration file on the fly.
-      const _ = new SourceFile({
-        filename: `${sourceFileJson.filename}.d.ts`,
-        url: `${sourceFileJson.url}.d.ts`,
-        mediaType: MediaType.TypeScript,
-        sourceCode: "export default any;"
-      });
-    }
     const sourceFile =
       SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
     sourceFile.cache(specifiers[i][0], referrer);
@@ -395,7 +374,7 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
       return ts.Extension.Json;
     case MediaType.Wasm:
       // Custom marker for Wasm type.
-      return WASM_MARKER;
+      return ts.Extension.Js;
     case MediaType.Unknown:
     default:
       throw TypeError("Cannot resolve extension.");
@@ -573,14 +552,6 @@ class Host implements ts.CompilerHost {
       if (!sourceFile) {
         return undefined;
       }
-      if (sourceFile.isWasm) {
-        // WASM import, use custom .d.ts declaration instead.
-        return {
-          resolvedFileName: `${sourceFile.url}.d.ts`,
-          isExternalLibraryImport: false,
-          extension: ts.Extension.Dts
-        };
-      }
       return {
         resolvedFileName: sourceFile.url,
         isExternalLibraryImport: specifier.startsWith(ASSETS),
@@ -757,3 +728,47 @@ window.compilerMain = function compilerMain(): void {
     workerClose();
   };
 };
+
+function base64ToUint8Array(data: string): Uint8Array {
+  const binString = window.atob(data);
+  const size = binString.length;
+  const bytes = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    bytes[i] = binString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+window.wasmCompilerMain = function wasmCompilerMain(): void {
+  // workerMain should have already been called since a compiler is a worker.
+  window.onmessage = async ({
+    data: binary
+  }: {
+    data: string;
+  }): Promise<void> => {
+    const buffer = base64ToUint8Array(binary);
+    // @ts-ignore
+    const compiled = await WebAssembly.compile(buffer);
+
+    util.log(">>> WASM compile start");
+
+    const importList = Array.from(
+      // @ts-ignore
+      new Set(WebAssembly.Module.imports(compiled).map(({ module }) => module))
+    );
+    const exportList = Array.from(
+      // @ts-ignore
+      new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name))
+    );
+
+    postMessage({
+      importList,
+      exportList
+    });
+
+    util.log("<<< WASM compile end");
+
+    // The compiler isolate exits after a single message.
+    workerClose();
+  };
+};