Add example with WASM audio worklet

Author: lukaslihotzki

.github/workflows/main.yml

@@ -228,7 +228,7 @@ jobs:
         ln -snf `pwd`/target/debug/wasm-bindgen $(dirname `which cargo`)/wasm-bindgen
     - run: mv _package.json package.json && npm install && rm package.json
     - run: |
-        for dir in `ls examples | grep -v README | grep -v asm.js | grep -v raytrace | grep -v without-a-bundler | grep -v wasm-in-web-worker | grep -v websockets | grep -v webxr | grep -v deno | grep -v synchronous-instantiation`; do
+        for dir in `ls examples | grep -v README | grep -v asm.js | grep -v raytrace | grep -v without-a-bundler | grep -v wasm-in-web-worker | grep -v websockets | grep -v webxr | grep -v deno | grep -v synchronous-instantiation | grep -v wasm-audio-worklet`; do
           (cd examples/$dir &&
           ln -fs ../../node_modules . &&
           npm run build -- --output-path ../../exbuild/$dir) || exit 1;
@@ -255,6 +255,23 @@ jobs:
         name: examples2
         path: exbuild
 
+  build_wasm_audio_worklet:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: rustup default nightly-2022-07-10
+    - run: rustup target add wasm32-unknown-unknown
+    - run: rustup component add rust-src
+    - run: |
+        sed -i 's/python/#python/' examples/wasm-audio-worklet/build.sh
+        (cd examples/wasm-audio-worklet && ./build.sh)
+        mkdir exbuild
+        cp examples/wasm-audio-worklet/*.{js,html,wasm} exbuild
+    - uses: actions/upload-artifact@v2
+      with:
+        name: examples3
+        path: exbuild
+
   build_benchmarks:
     runs-on: ubuntu-latest
     steps:
@@ -394,6 +411,7 @@ jobs:
         mv target/doc gh-pages/api
         mv artifacts/examples1 gh-pages/exbuild
         mv artifacts/examples2 gh-pages/exbuild/raytrace-parallel
+        mv artifacts/examples3 gh-pages/exbuild/wasm-audio-worklet
         mv artifacts/benchmarks gh-pages/benchmarks
         tar czf gh-pages.tar.gz gh-pages
     - uses: actions/upload-artifact@v2

Cargo.toml

@@ -77,6 +77,7 @@ members = [
   "examples/raytrace-parallel",
   "examples/request-animation-frame",
   "examples/todomvc",
+  "examples/wasm-audio-worklet",
   "examples/wasm-in-wasm",
   "examples/wasm-in-wasm-imports",
   "examples/wasm-in-web-worker",

examples/wasm-audio-worklet/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "wasm-audio-worklet"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+console_log = "0.2.0"
+js-sys = "0.3.59"
+wasm-bindgen = "0.2.82"
+wasm-bindgen-futures = "0.4.32"
+web-sys = {version = "0.3.59", features = ["AudioContext", "AudioDestinationNode", "AudioWorklet", "AudioWorkletNode", "AudioWorkletNodeOptions", "Blob", "BlobPropertyBag", "Document", "HtmlInputElement", "HtmlLabelElement", "Url", "Window"]}
+
+[lib]
+crate-type = ["cdylib", "rlib"]

examples/wasm-audio-worklet/README.md

@@ -0,0 +1,28 @@
+# wasm-audio-worklet
+
+[View documentation for this example online][dox] or [View compiled example
+online][compiled]
+
+[dox]: https://rustwasm.github.io/docs/wasm-bindgen/examples/wasm-audio-worklet.html
+[compiled]: https://rustwasm.github.io/wasm-bindgen/exbuild/wasm-audio-worklet/
+
+You can build the example locally with:
+
+```
+$ ./build.sh
+```
+
+(or running the commands on Windows manually)
+
+and then visiting http://localhost:8080 in a browser should run the example!
+
+## dependent_module
+
+This example shows how libraries can create worklets with minimal extra effort
+for application developers. No extra scripts are needed.
+Just the build commands need to be changed to enable threads, and the server
+needs to send extra headers to enable site isolation.
+
+dependent_module relies on ES6 `import`, which is
+[currently unavailable](https://bugzilla.mozilla.org/show_bug.cgi?id=1572644)
+for worklets in Firefox. The example works in Chrome.

examples/wasm-audio-worklet/build.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -ex
+
+# A couple of steps are necessary to get this build working which makes it slightly
+# nonstandard compared to most other builds.
+#
+# * First, the Rust standard library needs to be recompiled with atomics
+#   enabled. to do that we use Cargo's unstable `-Zbuild-std` feature.
+#
+# * Next we need to compile everything with the `atomics` and `bulk-memory`
+#   features enabled, ensuring that LLVM will generate atomic instructions,
+#   shared memory, passive segments, etc.
+
+RUSTFLAGS='-C target-feature=+atomics,+bulk-memory,+mutable-globals' \
+  cargo build --target wasm32-unknown-unknown --release -Z build-std=std,panic_abort
+
+cargo run -p wasm-bindgen-cli -- \
+  ../../target/wasm32-unknown-unknown/release/wasm_audio_worklet.wasm \
+  --out-dir . \
+  --target web
+
+python3 server.py

examples/wasm-audio-worklet/index.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>WASM audio worklet</title>
+    </head>
+    <body>
+        <script type="module">
+            import init, {web_main} from "./wasm_audio_worklet.js";
+            async function run() {
+                await init();
+                web_main();
+            }
+            run();
+        </script>
+    </body>
+</html>

examples/wasm-audio-worklet/server.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+from http.server import HTTPServer, SimpleHTTPRequestHandler, test
+import sys
+
+class RequestHandler(SimpleHTTPRequestHandler):
+    def end_headers(self):
+        self.send_header('Cross-Origin-Opener-Policy', 'same-origin')
+        self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')
+        SimpleHTTPRequestHandler.end_headers(self)
+
+if __name__ == '__main__':
+    test(RequestHandler, HTTPServer, port=int(sys.argv[1]) if len(sys.argv) > 1 else 8000)

examples/wasm-audio-worklet/src/dependent_module.rs

@@ -0,0 +1,43 @@
+use js_sys::{Array, JsString};
+use wasm_bindgen::prelude::*;
+use web_sys::{Blob, BlobPropertyBag, Url};
+
+// This is a not-so-clean approach to get the current bindgen JS module URL
+// in Rust.
+#[wasm_bindgen]
+extern "C" {
+    #[wasm_bindgen]
+    type ImportMeta;
+
+    #[wasm_bindgen(method, getter)]
+    fn url(this: &ImportMeta) -> JsString;
+
+    #[wasm_bindgen(js_namespace = import, js_name = meta)]
+    static IMPORT_META: ImportMeta;
+}
+
+pub fn on_the_fly(code: &str) -> Result<String, JsValue> {
+    let header = format!(
+        "import init, * as bindgen from '{}';\n\n",
+        IMPORT_META.url(),
+    );
+    Url::create_object_url_with_blob(&Blob::new_with_str_sequence_and_options(
+        &Array::of2(&JsValue::from(header.as_str()), &JsValue::from(code)),
+        &BlobPropertyBag::new().type_("text/javascript"),
+    )?)
+}
+
+// dependent_module! takes a local file name to a JS module as input and
+// returns a URL to a slightly modified module in run time. This modified module
+// has an additional import statement in the header that imports the current
+// bindgen JS module under the `bindgen` alias.
+// How this URL is produced does not matter for the macro user. on_the_fly
+// creates a blob URL in run time. A better, more sophisticated solution
+// would add wasm_bindgen support to put such a module in pkg/ during build time
+// and return a relative URL to this file instead.
+#[macro_export]
+macro_rules! dependent_module {
+    ($file_name:expr) => {
+        crate::dependent_module::on_the_fly(include_str!($file_name))
+    };
+}

examples/wasm-audio-worklet/src/get_bindgen_transferable.js

@@ -0,0 +1,6 @@
+export function get_bindgen_transferable(o) {
+    return {
+        type: Object.getPrototypeOf(o).constructor.name,
+        ptr: o.ptr,
+    };
+}

examples/wasm-audio-worklet/src/gui.rs

@@ -0,0 +1,39 @@
+use crate::oscillator::Params;
+use wasm_bindgen::{closure::Closure, JsCast, JsValue};
+use web_sys::{AudioContext, HtmlInputElement, HtmlLabelElement};
+
+pub fn create_gui(params: &'static Params, ctx: AudioContext) {
+    let window = web_sys::window().unwrap();
+    let document = window.document().unwrap();
+    let body = document.body().unwrap();
+
+    let volume = add_slider(&document, &body, "Volume:").unwrap();
+    let frequency = add_slider(&document, &body, "Frequency:").unwrap();
+    volume.set_value("0");
+    frequency.set_min("20");
+    frequency.set_value("60");
+
+    let listener = Closure::<dyn FnMut(_)>::new(move |_: web_sys::Event| {
+        params.set_frequency(frequency.value().parse().unwrap());
+        params.set_volume(volume.value().parse().unwrap());
+        ctx.resume().unwrap();
+    })
+    .into_js_value();
+
+    body.add_event_listener_with_callback("input", listener.as_ref().unchecked_ref())
+        .unwrap();
+}
+
+fn add_slider(
+    document: &web_sys::Document,
+    body: &web_sys::HtmlElement,
+    name: &str,
+) -> Result<HtmlInputElement, JsValue> {
+    let input: HtmlInputElement = document.create_element("input")?.unchecked_into();
+    let label: HtmlLabelElement = document.create_element("label")?.unchecked_into();
+    input.set_type("range");
+    label.set_text_content(Some(name));
+    label.append_child(&input)?;
+    body.append_child(&label)?;
+    Ok(input)
+}

examples/wasm-audio-worklet/src/lib.rs

@@ -0,0 +1,20 @@
+mod dependent_module;
+mod gui;
+mod oscillator;
+mod wasm_audio;
+
+use gui::create_gui;
+use oscillator::{Oscillator, Params};
+use wasm_audio::wasm_audio;
+use wasm_bindgen::prelude::*;
+
+#[wasm_bindgen]
+pub async fn web_main() {
+    // On the application level, audio worklet internals are abstracted by wasm_audio:
+    let params: &'static Params = Box::leak(Box::new(Params::default()));
+    let mut osc = Oscillator::new(&params);
+    let ctx = wasm_audio(Box::new(move |buf| osc.process(buf)))
+        .await
+        .unwrap();
+    create_gui(params, ctx);
+}

examples/wasm-audio-worklet/src/oscillator.rs

@@ -0,0 +1,54 @@
+// WASM audio processors can be implemented in Rust without knowing
+// about audio worklets.
+
+use std::sync::atomic::{AtomicU8, Ordering};
+
+// Let's implement a simple sine oscillator with variable frequency and volume.
+pub struct Oscillator {
+    params: &'static Params,
+    accumulator: u32,
+}
+
+impl Oscillator {
+    pub fn new(params: &'static Params) -> Self {
+        Self {
+            params,
+            accumulator: 0,
+        }
+    }
+}
+
+impl Oscillator {
+    pub fn process(&mut self, output: &mut [f32]) -> bool {
+        // This method is called in the audio process thread.
+        // All imports are set, so host functionality available in worklets
+        // (for example, logging) can be used:
+        // `web_sys::console::log_1(&JsValue::from(output.len()));`
+        // Note that currently TextEncoder and TextDecoder are stubs, so passing
+        // strings may not work in this thread.
+        for a in output {
+            let frequency = self.params.frequency.load(Ordering::Relaxed);
+            let volume = self.params.volume.load(Ordering::Relaxed);
+            self.accumulator += u32::from(frequency);
+            *a = (self.accumulator as f32 / 512.).sin() * (volume as f32 / 100.);
+        }
+        true
+    }
+}
+
+#[derive(Default)]
+pub struct Params {
+    // Use atomics for parameters so they can be set in the main thread and
+    // fetched by the audio process thread without further synchronization.
+    frequency: AtomicU8,
+    volume: AtomicU8,
+}
+
+impl Params {
+    pub fn set_frequency(&self, frequency: u8) {
+        self.frequency.store(frequency, Ordering::Relaxed);
+    }
+    pub fn set_volume(&self, volume: u8) {
+        self.volume.store(volume, Ordering::Relaxed);
+    }
+}

examples/wasm-audio-worklet/src/polyfill.js

@@ -0,0 +1,18 @@
+if (!globalThis.TextDecoder) {
+    globalThis.TextDecoder = class TextDecoder {
+        decode() {
+            return "";
+        }
+    };
+}
+
+if (!globalThis.TextEncoder) {
+    globalThis.TextEncoder = class TextEncoder {
+        encode() {
+            return new Uint8Array(0);
+        }
+    };
+}
+
+export function nop() {
+}

examples/wasm-audio-worklet/src/wasm_audio.rs

@@ -0,0 +1,60 @@
+use crate::dependent_module;
+use wasm_bindgen::prelude::*;
+use wasm_bindgen::JsValue;
+use wasm_bindgen_futures::JsFuture;
+use web_sys::{AudioContext, AudioWorkletNode, AudioWorkletNodeOptions};
+
+#[wasm_bindgen]
+struct WasmAudioProcessor(Box<dyn FnMut(&mut [f32]) -> bool>);
+
+#[wasm_bindgen]
+impl WasmAudioProcessor {
+    pub fn process(&mut self, buf: &mut [f32]) -> bool {
+        self.0(buf)
+    }
+}
+
+// Use wasm_audio if you have a single wasm audio processor in your application
+// whose samples should be played directly.
+pub async fn wasm_audio(
+    process: Box<dyn FnMut(&mut [f32]) -> bool>,
+) -> Result<AudioContext, JsValue> {
+    let ctx = AudioContext::new()?;
+    prepare_wasm_audio(&ctx).await?;
+    let node = wasm_audio_node(&ctx, process)?;
+    node.connect_with_audio_node(&ctx.destination())?;
+    Ok(ctx)
+}
+
+// wasm_audio_node creates an AudioWorkletNode running a wasm audio processor.
+// Remember to call prepare_wasm_audio once on your context before calling
+// this function.
+pub fn wasm_audio_node(
+    ctx: &AudioContext,
+    process: Box<dyn FnMut(&mut [f32]) -> bool>,
+) -> Result<AudioWorkletNode, JsValue> {
+    AudioWorkletNode::new_with_options(
+        &ctx,
+        "WasmProcessor",
+        &AudioWorkletNodeOptions::new().processor_options(Some(&js_sys::Array::of3(
+            &wasm_bindgen::module(),
+            &wasm_bindgen::memory(),
+            &JsValue::from(WasmAudioProcessor(process)),
+        ))),
+    )
+}
+
+pub async fn prepare_wasm_audio(ctx: &AudioContext) -> Result<(), JsValue> {
+    nop();
+    let mod_url = dependent_module!("worklet_async.js")?;
+    JsFuture::from(ctx.audio_worklet()?.add_module(&mod_url)?).await?;
+    Ok(())
+}
+
+// TextEncoder and TextDecoder are not available in Audio Worklets, but there
+// is a dirty workaround: Import polyfill.js to install stub implementations
+// of these classes in globalThis.
+#[wasm_bindgen(module = "/src/polyfill.js")]
+extern "C" {
+    fn nop();
+}