feat: add memoization to when converting a graph into a tree

Author: admons

src/legacy/index.ts

@@ -1,6 +1,6 @@
 import * as crypto from 'crypto';
 import * as types from '../core/types';
-import { DepGraphBuilder } from '../core/builder';
+import { DepGraphBuilder } from '..';
 import { EventLoopSpinner } from './event-loop-spinner';
 
 import objectHash = require('object-hash');
@@ -286,7 +286,11 @@ async function buildSubtree(
   nodeId: string,
   eventLoopSpinner: EventLoopSpinner,
   maybeDeduplicationSet: Set<string> | null | false = null, // false = disabled; null = not in deduplication scope yet
+  memoizationMap: Map<string, DepTree> = new Map(),
 ): Promise<DepTree> {
+  if (!maybeDeduplicationSet && memoizationMap.has(nodeId)) {
+    return memoizationMap.get(nodeId)!;
+  }
   const isRoot = nodeId === depGraph.rootNodeId;
   const nodePkg = depGraph.getNodePkg(nodeId);
   const nodeInfo = depGraph.getNode(nodeId);
@@ -302,6 +306,7 @@ async function buildSubtree(
 
   const depInstanceIds = depGraph.getNodeDepsNodeIds(nodeId);
   if (!depInstanceIds || depInstanceIds.length === 0) {
+    memoizationMap.set(nodeId, depTree);
     return depTree;
   }
 
@@ -326,6 +331,7 @@ async function buildSubtree(
       depInstId,
       eventLoopSpinner,
       maybeDeduplicationSet,
+      memoizationMap,
     );
     if (!subtree) {
       continue;
@@ -341,6 +347,7 @@ async function buildSubtree(
   if (eventLoopSpinner.isStarving()) {
     await eventLoopSpinner.spin();
   }
+  memoizationMap.set(nodeId, depTree);
   return depTree;
 }
 

test/core/stress.test.ts

@@ -1,4 +1,5 @@
 import * as depGraphLib from '../../src';
+import { graphToDepTree } from '../../src/legacy';
 
 const dependencyName = 'needle';
 
@@ -10,11 +11,14 @@ async function generateLargeGraph(width: number) {
   const rootNodeId = 'root-node';
 
   const deepDependency = { name: dependencyName, version: '1.2.3' };
+  const deepDependency2 = { name: dependencyName + 2, version: '1.2.3' };
 
   builder.addPkgNode(deepDependency, dependencyName);
+  builder.addPkgNode(deepDependency2, deepDependency2.name);
   builder.connectDep(rootNodeId, dependencyName);
+  builder.connectDep(deepDependency.name, deepDependency2.name);
 
-  for (let j = 0; j < width; j++) {
+  for (let j = 0; j < width / 2; j++) {
     const shallowName = `id-${j}`;
     const shallowDependency = { name: shallowName, version: '1.2.3' };
 
@@ -23,17 +27,34 @@ async function generateLargeGraph(width: number) {
     builder.connectDep(shallowName, dependencyName);
   }
 
+  for (let j = 0; j < width / 2; j++) {
+    const shallowName = `second-${j}`;
+    const shallowDependency = { name: shallowName, version: '1.2.3' };
+
+    builder.addPkgNode(shallowDependency, shallowName);
+    builder.connectDep(deepDependency2.name, shallowName);
+  }
+
   return builder.build();
 }
 
 describe('stress tests', () => {
+  let graph;
+  beforeAll(async () => {
+    graph = await generateLargeGraph(125000);
+  });
   test('pkgPathsToRoot() does not cause stack overflow for large dep-graphs', async () => {
-    const graph = await generateLargeGraph(125000);
-
     const result = graph.pkgPathsToRoot({
       name: dependencyName,
       version: '1.2.3',
     });
     expect(result).toBeDefined();
   });
+
+  test('graphToDepTree() with memoization (without deduplicateWithinTopLevelDeps) succeed for large dep-graphs', async () => {
+    const depTree = await graphToDepTree(graph, 'gomodules', {
+      deduplicateWithinTopLevelDeps: false,
+    });
+    expect(depTree).toBeDefined();
+  });
 });