Merge pull request #73 from snyk/feat/memoization-in-graph-to-dep-tree

feat: add memoization when converting a graph into a tree

Author: admons

src/legacy/index.ts

@@ -285,8 +285,12 @@ async function buildSubtree(
   depGraph: types.DepGraphInternal,
   nodeId: string,
   eventLoopSpinner: EventLoopSpinner,
-  maybeDeduplicationSet: Set<string> | null | false = null, // false = disabled; null = not in deduplication scope yet
+  maybeDeduplicationSet: Set<string> | null | false = false, // false = disabled; null = not in deduplication scope yet
+  memoizationMap: Map<string, DepTree> = new Map(),
 ): Promise<DepTree> {
+  if (!maybeDeduplicationSet && memoizationMap.has(nodeId)) {
+    return memoizationMap.get(nodeId)!;
+  }
   const isRoot = nodeId === depGraph.rootNodeId;
   const nodePkg = depGraph.getNodePkg(nodeId);
   const nodeInfo = depGraph.getNode(nodeId);
@@ -302,6 +306,7 @@ async function buildSubtree(
 
   const depInstanceIds = depGraph.getNodeDepsNodeIds(nodeId);
   if (!depInstanceIds || depInstanceIds.length === 0) {
+    memoizationMap.set(nodeId, depTree);
     return depTree;
   }
 
@@ -326,6 +331,7 @@ async function buildSubtree(
       depInstId,
       eventLoopSpinner,
       maybeDeduplicationSet,
+      memoizationMap,
     );
     if (!subtree) {
       continue;
@@ -341,6 +347,7 @@ async function buildSubtree(
   if (eventLoopSpinner.isStarving()) {
     await eventLoopSpinner.spin();
   }
+  memoizationMap.set(nodeId, depTree);
   return depTree;
 }
 

test/legacy/stress.test.ts

@@ -0,0 +1,50 @@
+import * as depGraphLib from '../../src';
+import { graphToDepTree } from '../../src/legacy';
+
+const dependencyName = 'needle';
+
+async function generateLargeGraph(width: number) {
+  const builder = new depGraphLib.DepGraphBuilder(
+    { name: 'npm' },
+    { name: 'root', version: '1.2.3' },
+  );
+  const rootNodeId = 'root-node';
+
+  const deepDependency = { name: dependencyName, version: '1.2.3' };
+  const deepDependency2 = { name: dependencyName + 2, version: '1.2.3' };
+
+  builder.addPkgNode(deepDependency, dependencyName);
+  builder.addPkgNode(deepDependency2, deepDependency2.name);
+  builder.connectDep(rootNodeId, dependencyName);
+  builder.connectDep(deepDependency.name, deepDependency2.name);
+
+  for (let j = 0; j < width / 2; j++) {
+    const shallowName = `id-${j}`;
+    const shallowDependency = { name: shallowName, version: '1.2.3' };
+
+    builder.addPkgNode(shallowDependency, shallowName);
+    builder.connectDep(rootNodeId, shallowName);
+    builder.connectDep(shallowName, dependencyName);
+  }
+
+  for (let j = 0; j < width / 2; j++) {
+    const shallowName = `second-${j}`;
+    const shallowDependency = { name: shallowName, version: '1.2.3' };
+
+    builder.addPkgNode(shallowDependency, shallowName);
+    builder.connectDep(deepDependency2.name, shallowName);
+  }
+
+  return builder.build();
+}
+
+describe('stress tests', () => {
+  test('graphToDepTree() with memoization (without deduplicateWithinTopLevelDeps) succeed for large dep-graphs', async () => {
+    const graph = await generateLargeGraph(125000);
+
+    const depTree = await graphToDepTree(graph, 'gomodules', {
+      deduplicateWithinTopLevelDeps: false,
+    });
+    expect(depTree).toBeDefined();
+  });
+});