perf(utils): add benchmarks for file system walk and glob libs (#514)

Author: BioPhoton

package-lock.json

@@ -67,6 +67,7 @@
     "@commitlint/config-nx-scopes": "^18.4.3",
     "@commitlint/cz-commitlint": "^17.7.1",
     "@jscutlery/semver": "^4.1.0",
+    "@nodelib/fs.walk": "^2.0.0",
     "@nx/devkit": "17.1.3",
     "@nx/esbuild": "17.1.3",
     "@nx/eslint-plugin": "17.1.3",
@@ -110,6 +111,9 @@
     "eslint-plugin-sonarjs": "^0.22.0",
     "eslint-plugin-unicorn": "^48.0.1",
     "eslint-plugin-vitest": "^0.3.8",
+    "fast-glob": "^3.3.2",
+    "glob": "^10.3.10",
+    "globby": "^14.0.1",
     "husky": "^8.0.0",
     "inquirer": "^8.2.6",
     "jsdom": "~22.1.0",

package.json

@@ -0,0 +1,27 @@
+import { type Entry, walkStream } from '@nodelib/fs.walk';
+import { CrawlFileSystemOptions } from '../../src';
+
+// from https://github.com/webpro/knip/pull/426
+export function crawlFileSystemFsWalk<T = string>(
+  options: CrawlFileSystemOptions<T>,
+): Promise<T[]> {
+  const { directory } = options;
+
+  return new Promise((resolve, reject) => {
+    const result: T[] = [];
+    const stream = walkStream(directory);
+
+    stream.on('data', (entry: Entry) => {
+      // eslint-disable-next-line functional/immutable-data
+      result.push(entry.path as T);
+    });
+
+    stream.on('error', error => {
+      reject(error);
+    });
+
+    stream.on('end', () => {
+      resolve(result);
+    });
+  });
+}

packages/utils/perf/crawl-file-system/fs-walk.ts

@@ -4,7 +4,7 @@ import {
   CrawlFileSystemOptions,
   crawlFileSystem,
 } from '../../src/lib/file-system';
-import { crawlFileSystemOptimized0 } from './optimized0';
+import { crawlFileSystemFsWalk } from './fs-walk';
 
 const PROCESS_ARGUMENT_TARGET_DIRECTORY =
   process.argv
@@ -54,7 +54,7 @@ const options = {
   pattern: PATTERN,
 };
 suite.add('Base', wrapWithDefer(crawlFileSystem));
-suite.add('Optimized 0', wrapWithDefer(crawlFileSystemOptimized0));
+suite.add('nodelib.fsWalk', wrapWithDefer(crawlFileSystemFsWalk));
 
 // ==================
 

packages/utils/perf/crawl-file-system/index.ts

@@ -0,0 +1,5 @@
+import * as fg from 'fast-glob';
+
+export function fastGlob(pattern: string[]): Promise<string[]> {
+  return fg.async(pattern);
+}

packages/utils/perf/crawl-file-system/optimized0.ts

@@ -0,0 +1,5 @@
+import { glob as g } from 'glob';
+
+export function glob(pattern: string[]): Promise<string[]> {
+  return g(pattern);
+}

packages/utils/perf/glob/fast-glob.ts

@@ -0,0 +1,5 @@
+import { globby as g } from 'globby';
+
+export function globby(pattern: string[]): Promise<string[]> {
+  return g(pattern);
+}

packages/utils/perf/glob/glob.ts

@@ -0,0 +1,92 @@
+import * as Benchmark from 'benchmark';
+import { join } from 'node:path';
+import { fastGlob } from './fast-glob';
+import { glob } from './glob';
+import { globby } from './globby';
+
+const suite = new Benchmark.Suite('report-scoring');
+
+const BASE_PATH = join(
+  process.cwd(),
+  '..',
+  '..',
+  '..',
+  'node_modules',
+  '**/*.js',
+);
+
+// ==================
+
+const start = performance.now();
+
+// Add listener
+const listeners = {
+  cycle: function (event: Benchmark.Event) {
+    console.info(String(event.target));
+  },
+  complete: () => {
+    if (typeof suite.filter === 'function') {
+      console.info(' ');
+      console.info(
+        `Total Duration: ${((performance.now() - start) / 1000).toFixed(
+          2,
+        )} sec`,
+      );
+      console.info(`Fastest is ${String(suite.filter('fastest').map('name'))}`);
+    }
+  },
+};
+
+// ==================
+
+// Add tests
+const pattern = [BASE_PATH];
+suite.add('glob', wrapWithDefer(glob));
+suite.add('globby', wrapWithDefer(globby));
+suite.add('fastGlob', wrapWithDefer(fastGlob));
+
+// ==================
+
+// Add Listener
+Object.entries(listeners).forEach(([name, fn]) => {
+  suite.on(name, fn);
+});
+
+// ==================
+
+console.info('You can adjust the test with the following arguments:');
+console.info(`pattern      glob pattern of test      --pattern=${BASE_PATH}`);
+console.info(' ');
+console.info('Start benchmark...');
+console.info(' ');
+
+suite.run({
+  async: true,
+});
+
+// ==============================================================
+
+function wrapWithDefer(asyncFn: (pattern: string[]) => Promise<string[]>) {
+  const logged: Record<string, boolean> = {};
+  return {
+    defer: true, // important for async functions
+    fn: function (deferred: { resolve: () => void }) {
+      return asyncFn(pattern)
+        .catch(() => [])
+        .then((result: unknown[]) => {
+          if (result.length === 0) {
+            throw new Error(`Result length is ${result.length}`);
+          } else {
+            if (!logged[asyncFn.name]) {
+              // eslint-disable-next-line functional/immutable-data
+              logged[asyncFn.name] = true;
+              // eslint-disable-next-line no-console
+              console.log(`${asyncFn.name} found ${result.length} files`);
+            }
+            deferred.resolve();
+          }
+          return void 0;
+        });
+    },
+  };
+}