(wasm) New approach for parameterized rules (#519)

Implement a new approach for parameterized rules:
- Analyze the grammar and create specialized versions of parameterized rules — one for every possible set of actual parameters
- The specialized rules are effectively closures: they capture a particular set of concrete parameters, so they can be invoked without arguments. This also means we can memoize them.
- This allows us to remove the runtime closure creation code; it allows us to support arbitrary numbers of parameters; and it eliminates the runtime functions for evaluating rules with fixed numbers of parameters (`evalApply1`, etc.)

Author: pdubroy

packages/miniohm-js/index.js

@@ -1,4 +1,4 @@
-/* global TextDecoder, TextEncoder, WebAssembly */
+/* global process, TextDecoder, TextEncoder, WebAssembly */
 
 const WASM_PAGE_SIZE = 64 * 1024;
 const INPUT_BUFFER_OFFSET = WASM_PAGE_SIZE;
@@ -88,6 +88,7 @@ export class WasmMatcher {
   }
 
   match() {
+    if (process.env.OHM_DEBUG === '1') debugger; // eslint-disable-line no-debugger
     return this._instance.exports.match(0);
   }
 

packages/wasm/TODO.md

@@ -6,9 +6,11 @@
 - [x] NonterminalNodes should keep track of the rule
 - [ ] When iteration contains a sequence, the children are flattened into the iter node.
 - [x] Basic parameterized rules
-- [ ] Parameterized rules with >3 params
+- [x] Parameterized rules with >3 params
 - [x] Parameters that aren't terminals
-- [ ] Memoization for parameterized rules
+- [x] Memoization for parameterized rules
+- [ ] Avoid unnecessary dispatch in generalized rules
+- [ ] Avoid duplicate lifted rules.
 - [x] Support direct left recursion.
 - [ ] Handle left recursion detection at grammar parse time.
 - [x] Separate API for _creating_ the Wasm module from the WasmMatcher interface.
@@ -26,4 +28,3 @@
 
 - How to deal with matchLength in lookahead. In regular Ohm, lookahead _does_ bind things. But that is hard to square with the current CST representation, that stores only the matchLength. Because somehow the things inside a lookahead must consume nothing — but if you have `&("a" "b")`, the only way to make them consume nothing (in the current representation) is to rewrite the matchLength of the two terminal nodes.
   - Could we introduce a pseudo-node for lookahead? It could get transparently unpacked when walking the tree.
-- Memoization of parameterized rules: Alex suggested assigning memoization keys statically to unique applications

packages/wasm/runtime/ohmRuntime.ts

@@ -101,6 +101,15 @@ export function match(startRuleId: i32): Result {
   return call_indirect<Result>(ruleId);
 }
 
+export function evalApplyGeneralized(ruleId: i32, caseIdx: i32): Result {
+  const origPos = pos;
+  const origNumBindings = bindings.length;
+  if (call_indirect<Result>(ruleId, caseIdx)) {
+    return newNonterminalNode(origPos, pos, ruleId, origNumBindings);
+  }
+  return 0;
+}
+
 export function evalApplyNoMemo0(ruleId: i32): Result {
   const origPos = pos;
   const origNumBindings = bindings.length;
@@ -111,25 +120,6 @@ export function evalApplyNoMemo0(ruleId: i32): Result {
 }
 
 export function evalApply0(ruleId: i32): Result {
-  // Handle closures, which are an pointer w/ a flag in the high bit.
-  // TODO: Find a cleaner way of doing this.
-  if (ruleId & 0x80000000) {
-    const ptr = ruleId & 0x7fffffff;
-    ruleId = load<i32>(ptr, 0);
-    const argCount = load<i32>(ptr, 4);
-    const args: i32[] = [];
-    for (let i = 0; i < argCount; i++) {
-      args.push(load<i32>(ptr + i * 4, 8));
-    }
-    switch (argCount) {
-      case 0: return evalApply0(ruleId);
-      case 1: return evalApply1(ruleId, args[0]);
-      case 2: return evalApply2(ruleId, args[0], args[1]);
-      case 3: return evalApply3(ruleId, args[0], args[1], args[2]);
-    }
-    assert(false);
-  }
-
   let result = memoTableGet(pos, ruleId);
   if (result !== 0) {
     return useMemoizedResult(ruleId, result);
@@ -177,54 +167,6 @@ export function handleLeftRecursion(origPos: usize, ruleId: i32, origNumBindings
   return succeeded;
 }
 
-export function evalApply1(ruleId: i32, arg0: i32): Result {
-  // if (hasMemoizedResult(ruleId)) {
-  //   return useMemoizedResult(ruleId);
-  // }
-  const origPos = pos;
-  const origNumBindings = bindings.length;
-  let result: Result = FAIL;
-  const succeeded = call_indirect<Result>(ruleId, arg0);
-  if (succeeded) {
-    const numChildren = bindings.length - origNumBindings;
-    result = newNonterminalNode(origPos, pos, ruleId, origNumBindings);
-  }
-  // memoizeResult(origPos, ruleId, result);
-  return succeeded;
-}
-
-export function evalApply2(ruleId: i32, arg0: i32, arg1: i32): Result {
-  // if (hasMemoizedResult(ruleId)) {
-  //   return useMemoizedResult(ruleId);
-  // }
-  const origPos = pos;
-  const origNumBindings = bindings.length;
-  let result: Result = FAIL;
-  const succeeded = call_indirect<Result>(ruleId, arg0, arg1);
-  if (succeeded) {
-    const numChildren = bindings.length - origNumBindings;
-    result = newNonterminalNode(origPos, pos, ruleId, origNumBindings);
-  }
-  // memoizeResult(origPos, ruleId, result);
-  return succeeded;
-}
-
-export function evalApply3(ruleId: i32, arg0: i32, arg1: i32, arg2: i32): Result {
-  // if (hasMemoizedResult(ruleId)) {
-  //   return useMemoizedResult(ruleId);
-  // }
-  const origPos = pos;
-  const origNumBindings = bindings.length;
-  let result: Result = FAIL;
-  const succeeded = call_indirect<Result>(ruleId, arg0, arg1, arg2);
-  if (succeeded) {
-    const numChildren = bindings.length - origNumBindings;
-    result = newNonterminalNode(origPos, pos, ruleId, origNumBindings);
-  }
-  // memoizeResult(origPos, ruleId, result);
-  return succeeded;
-}
-
 export function newTerminalNode(startIdx: i32, endIdx: i32): usize {
   const ptr = heap.alloc(CST_NODE_OVERHEAD);
   cstSetCount(ptr, 0);

packages/wasm/scripts/bench.js

@@ -15,7 +15,7 @@ const inputs = {
   featuredProduct: readFileSync(join(datadir, '_featured-product.liquid'), 'utf-8'),
   footer: readFileSync(join(datadir, '_footer.liquid'), 'utf-8'),
   html5shiv: readFileSync(join(datadir, '_html5shiv-3.7.3.js'), 'utf-8'),
-  underscore: readFileSync(join(datadir, '_underscore-1.8.3.js'), 'utf-8')
+  underscore: readFileSync(join(datadir, '_underscore-1.8.3.js'), 'utf-8'),
 };
 
 function checkOk(val) {
@@ -37,7 +37,7 @@ function benchWithSetup(name, setupFn, benchFn) {
   bench(name, function* () {
     yield {
       [0]: setupFn,
-      bench: benchFn
+      bench: benchFn,
     };
   });
 }
@@ -87,4 +87,4 @@ group('LiquidHTML: footer.liquid', () => {
   });
 });
 
-await run();
+(async () => await run())();