Skip to content

Commit 8e56cda

Browse files
authored
Rewrite wasm-bindgen with updated interface types proposal (#1882)
This commit is a pretty large scale rewrite of the internals of wasm-bindgen. No user-facing changes are expected as a result of this PR, but due to the scale of changes here it's likely inevitable that at least something will break. I'm hoping to get more testing in though before landing! The purpose of this PR is to update wasm-bindgen to the current state of the interface types proposal. The wasm-bindgen tool was last updated when it was still called "WebIDL bindings" so it's been awhile! All support is now based on https://github.com/bytecodealliance/wasm-interface-types which defines parsers/binary format/writers/etc for wasm-interface types. This is a pretty massive PR and unfortunately can't really be split up any more afaik. I don't really expect realistic review of all the code here (or commits), but some high-level changes are: * Interface types now consists of a set of "adapter functions". The IR in wasm-bindgen is modeled the same way not. * Each adapter function has a list of instructions, and these instructions work at a higher level than wasm itself, for example with strings. * The wasm-bindgen tool has a suite of instructions which are specific to it and not present in the standard. (like before with webidl bindings) * The anyref/multi-value transformations are now greatly simplified. They're simply "optimization passes" over adapter functions, removing instructions that are otherwise present. This way we don't have to juggle so much all over the place, and instructions always have the same meaning.
1 parent df34cf8 commit 8e56cda

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+5742
-4992
lines changed

azure-pipelines.yml

+4
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,10 @@ jobs:
9696
displayName: "wasm-bindgen-cli-support tests"
9797
- script: cargo test -p wasm-bindgen-cli
9898
displayName: "wasm-bindgen-cli tests"
99+
- script: cargo test -p wasm-bindgen-anyref-xform
100+
displayName: "wasm-bindgen-anyref-xform tests"
101+
- script: cargo test -p wasm-bindgen-multi-value-xform
102+
displayName: "wasm-bindgen-multi-value-xform tests"
99103

100104
- job: test_web_sys
101105
displayName: "Run web-sys crate tests"

crates/anyref-xform/Cargo.toml

+11-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,14 @@ edition = '2018'
1313

1414
[dependencies]
1515
anyhow = "1.0"
16-
walrus = "0.13.0"
16+
walrus = "0.14.0"
17+
18+
[dev-dependencies]
19+
rayon = "1.0"
20+
wasmprinter = "0.2"
21+
wast = "3.0"
22+
wat = "1.0"
23+
24+
[[test]]
25+
name = "all"
26+
harness = false

crates/anyref-xform/src/lib.rs

+18-4
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ pub struct Context {
4444
table: Option<TableId>,
4545
}
4646

47+
pub struct Meta {
48+
pub table: TableId,
49+
pub alloc: Option<FunctionId>,
50+
pub drop_slice: Option<FunctionId>,
51+
}
52+
4753
struct Transform<'a> {
4854
cx: &'a mut Context,
4955

@@ -161,7 +167,7 @@ impl Context {
161167
})
162168
}
163169

164-
pub fn run(&mut self, module: &mut Module) -> Result<(), Error> {
170+
pub fn run(&mut self, module: &mut Module) -> Result<Meta, Error> {
165171
let table = self.table.unwrap();
166172

167173
// Inject a stack pointer global which will be used for managing the
@@ -171,6 +177,7 @@ impl Context {
171177

172178
let mut heap_alloc = None;
173179
let mut heap_dealloc = None;
180+
let mut drop_slice = None;
174181

175182
// Find exports of some intrinsics which we only need for a runtime
176183
// implementation.
@@ -182,7 +189,8 @@ impl Context {
182189
match export.name.as_str() {
183190
"__wbindgen_anyref_table_alloc" => heap_alloc = Some(f),
184191
"__wbindgen_anyref_table_dealloc" => heap_dealloc = Some(f),
185-
_ => {}
192+
"__wbindgen_drop_anyref_slice" => drop_slice = Some(f),
193+
_ => continue,
186194
}
187195
}
188196
let mut clone_ref = None;
@@ -226,7 +234,13 @@ impl Context {
226234
heap_dealloc,
227235
stack_pointer,
228236
}
229-
.run(module)
237+
.run(module)?;
238+
239+
Ok(Meta {
240+
table,
241+
alloc: heap_alloc,
242+
drop_slice,
243+
})
230244
}
231245
}
232246

@@ -619,7 +633,7 @@ impl Transform<'_> {
619633
// with a fresh type we've been calculating so far. Give the function a
620634
// nice name for debugging and then we're good to go!
621635
let id = builder.finish(params, funcs);
622-
let name = format!("{}_anyref_shim", name);
636+
let name = format!("{} anyref shim", name);
623637
funcs.get_mut(id).name = Some(name);
624638
self.shims.insert(id);
625639
Ok((id, anyref_ty))

crates/anyref-xform/tests/all.rs

+258
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
//! A small test framework to execute a test function over all files in a
2+
//! directory.
3+
//!
4+
//! Each file in the directory has its own `CHECK-ALL` annotation indicating the
5+
//! expected output of the test. That can be automatically updated with
6+
//! `BLESS=1` in the environment. Otherwise the test are checked against the
7+
//! listed expectation.
8+
9+
use anyhow::{anyhow, bail, Context, Result};
10+
use rayon::prelude::*;
11+
use std::env;
12+
use std::fs;
13+
use std::path::{Path, PathBuf};
14+
use wast::parser::{Parse, Parser};
15+
16+
fn main() {
17+
run("tests".as_ref(), runtest);
18+
}
19+
20+
fn runtest(test: &Test) -> Result<String> {
21+
let wasm = wat::parse_file(&test.file)?;
22+
let mut walrus = walrus::Module::from_buffer(&wasm)?;
23+
let mut cx = wasm_bindgen_anyref_xform::Context::default();
24+
cx.prepare(&mut walrus)?;
25+
for directive in test.directives.iter() {
26+
match &directive.kind {
27+
DirectiveKind::Export(name) => {
28+
let export = walrus
29+
.exports
30+
.iter()
31+
.find(|e| e.name == *name)
32+
.ok_or_else(|| anyhow!("failed to find export"))?;
33+
cx.export_xform(export.id(), &directive.args, directive.ret_anyref);
34+
}
35+
DirectiveKind::Import(module, field) => {
36+
let import = walrus
37+
.imports
38+
.iter()
39+
.find(|e| e.module == *module && e.name == *field)
40+
.ok_or_else(|| anyhow!("failed to find export"))?;
41+
cx.import_xform(import.id(), &directive.args, directive.ret_anyref);
42+
}
43+
DirectiveKind::Table(idx) => {
44+
cx.table_element_xform(*idx, &directive.args, directive.ret_anyref);
45+
}
46+
}
47+
}
48+
cx.run(&mut walrus)?;
49+
walrus::passes::gc::run(&mut walrus);
50+
let printed = wasmprinter::print_bytes(&walrus.emit_wasm())?;
51+
Ok(printed)
52+
}
53+
54+
fn run(dir: &Path, run: fn(&Test) -> Result<String>) {
55+
let mut tests = Vec::new();
56+
find_tests(dir, &mut tests);
57+
let filter = std::env::args().nth(1);
58+
59+
let bless = env::var("BLESS").is_ok();
60+
let tests = tests
61+
.iter()
62+
.filter(|test| {
63+
if let Some(filter) = &filter {
64+
if let Some(s) = test.file_name().and_then(|s| s.to_str()) {
65+
if !s.contains(filter) {
66+
return false;
67+
}
68+
}
69+
}
70+
true
71+
})
72+
.collect::<Vec<_>>();
73+
74+
println!("\nrunning {} tests\n", tests.len());
75+
76+
let errors = tests
77+
.par_iter()
78+
.filter_map(|test| run_test(test, bless, run).err())
79+
.collect::<Vec<_>>();
80+
81+
if !errors.is_empty() {
82+
for msg in errors.iter() {
83+
eprintln!("error: {:?}", msg);
84+
}
85+
86+
panic!("{} tests failed", errors.len())
87+
}
88+
89+
println!("test result: ok. {} passed\n", tests.len());
90+
}
91+
92+
fn run_test(test: &Path, bless: bool, run: fn(&Test) -> anyhow::Result<String>) -> Result<()> {
93+
(|| -> Result<_> {
94+
let expected = Test::from_file(test)?;
95+
let actual = run(&expected)?;
96+
expected.check(&actual, bless)?;
97+
Ok(())
98+
})()
99+
.context(format!("test failed - {}", test.display()))?;
100+
Ok(())
101+
}
102+
103+
fn find_tests(path: &Path, tests: &mut Vec<PathBuf>) {
104+
for f in path.read_dir().unwrap() {
105+
let f = f.unwrap();
106+
if f.file_type().unwrap().is_dir() {
107+
find_tests(&f.path(), tests);
108+
continue;
109+
}
110+
match f.path().extension().and_then(|s| s.to_str()) {
111+
Some("wat") => {}
112+
_ => continue,
113+
}
114+
tests.push(f.path());
115+
}
116+
}
117+
118+
struct Test {
119+
file: PathBuf,
120+
directives: Vec<Directive>,
121+
assertion: Option<String>,
122+
}
123+
124+
struct Directive {
125+
args: Vec<(usize, bool)>,
126+
ret_anyref: bool,
127+
kind: DirectiveKind,
128+
}
129+
130+
enum DirectiveKind {
131+
Import(String, String),
132+
Export(String),
133+
Table(u32),
134+
}
135+
136+
impl Test {
137+
fn from_file(path: &Path) -> Result<Test> {
138+
let contents = fs::read_to_string(path)?;
139+
let mut iter = contents.lines();
140+
let mut assertion = None;
141+
let mut directives = Vec::new();
142+
while let Some(line) = iter.next() {
143+
if line.starts_with("(; CHECK-ALL:") {
144+
let mut pattern = String::new();
145+
while let Some(line) = iter.next() {
146+
if line == ";)" {
147+
break;
148+
}
149+
pattern.push_str(line);
150+
pattern.push_str("\n");
151+
}
152+
while pattern.ends_with("\n") {
153+
pattern.pop();
154+
}
155+
if iter.next().is_some() {
156+
bail!("CHECK-ALL must be at the end of the file");
157+
}
158+
assertion = Some(pattern);
159+
continue;
160+
}
161+
162+
if !line.starts_with(";; @xform") {
163+
continue;
164+
}
165+
let directive = &line[9..];
166+
let buf = wast::parser::ParseBuffer::new(directive)?;
167+
directives.push(wast::parser::parse::<Directive>(&buf)?);
168+
}
169+
Ok(Test {
170+
file: path.to_path_buf(),
171+
directives,
172+
assertion,
173+
})
174+
}
175+
176+
fn check(&self, output: &str, bless: bool) -> Result<()> {
177+
if bless {
178+
update_output(&self.file, output)
179+
} else if let Some(pattern) = &self.assertion {
180+
if output == pattern {
181+
return Ok(());
182+
}
183+
bail!(
184+
"expected\n {}\n\nactual\n {}",
185+
pattern.replace("\n", "\n "),
186+
output.replace("\n", "\n ")
187+
);
188+
} else {
189+
bail!(
190+
"no test assertions were found in this file, but you can \
191+
rerun tests with `BLESS=1` to automatically add assertions \
192+
to this file"
193+
);
194+
}
195+
}
196+
}
197+
198+
fn update_output(path: &Path, output: &str) -> Result<()> {
199+
let contents = fs::read_to_string(path)?;
200+
let start = contents.find("(; CHECK-ALL:").unwrap_or(contents.len());
201+
202+
let mut new_output = String::new();
203+
for line in output.lines() {
204+
new_output.push_str(line);
205+
new_output.push_str("\n");
206+
}
207+
let new = format!(
208+
"{}\n\n(; CHECK-ALL:\n{}\n;)\n",
209+
contents[..start].trim(),
210+
new_output.trim_end()
211+
);
212+
fs::write(path, new)?;
213+
Ok(())
214+
}
215+
216+
impl<'a> Parse<'a> for Directive {
217+
fn parse(parser: Parser<'a>) -> wast::parser::Result<Self> {
218+
use wast::kw;
219+
wast::custom_keyword!(anyref_owned);
220+
wast::custom_keyword!(anyref_borrowed);
221+
wast::custom_keyword!(other);
222+
223+
let kind = if parser.peek::<kw::import>() {
224+
parser.parse::<kw::import>()?;
225+
DirectiveKind::Import(parser.parse()?, parser.parse()?)
226+
} else if parser.peek::<kw::export>() {
227+
parser.parse::<kw::export>()?;
228+
DirectiveKind::Export(parser.parse()?)
229+
} else {
230+
parser.parse::<kw::table>()?;
231+
DirectiveKind::Table(parser.parse()?)
232+
};
233+
let mut args = Vec::new();
234+
parser.parens(|p| {
235+
let mut i = 0;
236+
while !p.is_empty() {
237+
if parser.peek::<anyref_owned>() {
238+
parser.parse::<anyref_owned>()?;
239+
args.push((i, true));
240+
} else if parser.peek::<anyref_borrowed>() {
241+
parser.parse::<anyref_borrowed>()?;
242+
args.push((i, false));
243+
} else {
244+
parser.parse::<other>()?;
245+
}
246+
i += 1;
247+
}
248+
Ok(())
249+
})?;
250+
251+
let ret_anyref = parser.parse::<Option<anyref_owned>>()?.is_some();
252+
Ok(Directive {
253+
args,
254+
ret_anyref,
255+
kind,
256+
})
257+
}
258+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
;; @xform export "foo" (anyref_owned)
2+
3+
(module
4+
(func $foo (export "foo") (param i32))
5+
(func $alloc (export "__wbindgen_anyref_table_alloc") (result i32)
6+
i32.const 0)
7+
(func $dealloc (export "__wbindgen_anyref_table_dealloc") (param i32))
8+
)
9+
10+
(; CHECK-ALL:
11+
(module
12+
(type (;0;) (func (result i32)))
13+
(type (;1;) (func (param i32)))
14+
(type (;2;) (func (param anyref)))
15+
(func $foo anyref shim (type 2) (param anyref)
16+
(local i32)
17+
call $alloc
18+
local.tee 1
19+
local.get 0
20+
table.set 0
21+
local.get 1
22+
call $foo)
23+
(func $alloc (type 0) (result i32)
24+
i32.const 0)
25+
(func $foo (type 1) (param i32))
26+
(func $dealloc (type 1) (param i32))
27+
(table (;0;) 32 anyref)
28+
(export "foo" (func $foo anyref shim))
29+
(export "__wbindgen_anyref_table_alloc" (func $alloc))
30+
(export "__wbindgen_anyref_table_dealloc" (func $dealloc)))
31+
;)

0 commit comments

Comments
 (0)