feat(utils): add file-system helper (#336)

Author: BioPhoton

packages/plugin-eslint/src/lib/runner/transform.ts

@@ -4,8 +4,8 @@ import { IssueSeverity } from '@code-pushup/models';
 import {
   compareIssueSeverity,
   countOccurrences,
-  formatCount,
   objectToEntries,
+  pluralizeToken,
 } from '@code-pushup/utils';
 import { ruleIdToSlug } from '../meta/hash';
 import type { LinterOutput } from './types';
@@ -43,7 +43,7 @@ function toAudit(slug: string, issues: LintIssue[]): AuditOutput {
   );
   const summaryText = objectToEntries(severityCounts)
     .sort((a, b) => -compareIssueSeverity(a[0], b[0]))
-    .map(([severity, count = 0]) => formatCount(count, severity))
+    .map(([severity, count = 0]) => pluralizeToken(severity, count))
     .join(', ');
   return {
     slug,

packages/utils/src/index.ts

@@ -7,45 +7,50 @@ export {
   executeProcess,
   objectToCliArgs,
 } from './lib/execute-process';
-export {
-  FileResult,
-  MultipleFileResults,
-  ensureDirectoryExists,
-  importEsmModule,
-  logMultipleFileResults,
-  pluginWorkDir,
-  readJsonFile,
-  readTextFile,
-  toUnixPath,
-} from './lib/file-system';
 export { getLatestCommit, git } from './lib/git';
-export { logMultipleResults } from './lib/log-results';
-export { NEW_LINE } from './lib/md';
 export { ProgressBar, getProgressBar } from './lib/progress';
-export {
-  isPromiseFulfilledResult,
-  isPromiseRejectedResult,
-} from './lib/promise-result';
 export {
   CODE_PUSHUP_DOMAIN,
   FOOTER_PREFIX,
   README_LINK,
   calcDuration,
   compareIssueSeverity,
-  formatBytes,
-  formatCount,
   loadReport,
 } from './lib/report';
 export { reportToMd } from './lib/report-to-md';
 export { reportToStdout } from './lib/report-to-stdout';
 export { ScoredReport, scoreReport } from './lib/scoring';
 export {
+  readJsonFile,
+  readTextFile,
+  toUnixPath,
+  ensureDirectoryExists,
+  FileResult,
+  MultipleFileResults,
+  logMultipleFileResults,
+  importEsmModule,
+  pluginWorkDir,
+  crawlFileSystem,
+  findLineNumberInText,
+} from './lib/file-system';
+export { verboseUtils } from './lib/verbose-utils';
+export {
+  toArray,
+  objectToKeys,
+  objectToEntries,
   countOccurrences,
   distinct,
-  objectToEntries,
-  objectToKeys,
-  pluralize,
-  slugify,
-  toArray,
 } from './lib/transformation';
-export { verboseUtils } from './lib/verbose-utils';
+export {
+  slugify,
+  pluralize,
+  pluralizeToken,
+  formatBytes,
+  formatDuration,
+} from './lib/formatting';
+export { NEW_LINE } from './lib/md';
+export { logMultipleResults } from './lib/log-results';
+export {
+  isPromiseFulfilledResult,
+  isPromiseRejectedResult,
+} from './lib/guards';

packages/utils/src/lib/file-system.ts

@@ -1,9 +1,9 @@
 import { type Options, bundleRequire } from 'bundle-require';
 import chalk from 'chalk';
-import { mkdir, readFile } from 'fs/promises';
+import { mkdir, readFile, readdir, stat } from 'fs/promises';
 import { join } from 'path';
+import { formatBytes } from './formatting';
 import { logMultipleResults } from './log-results';
-import { formatBytes } from './report';
 
 export function toUnixPath(
   path: string,
@@ -92,3 +92,42 @@ export async function importEsmModule<T = unknown>(
 export function pluginWorkDir(slug: string): string {
   return join('node_modules', '.code-pushup', slug);
 }
+
+export async function crawlFileSystem<T = string>(options: {
+  directory: string;
+  pattern?: string | RegExp;
+  fileTransform?: (filePath: string) => Promise<T> | T;
+}): Promise<T[]> {
+  const {
+    directory,
+    pattern,
+    fileTransform = (filePath: string) => filePath as T,
+  } = options;
+
+  const files = await readdir(directory);
+  const promises = files.map(async (file): Promise<T | T[]> => {
+    const filePath = join(directory, file);
+    const stats = await stat(filePath);
+
+    if (stats.isDirectory()) {
+      return crawlFileSystem({ directory: filePath, pattern, fileTransform });
+    }
+    if (stats.isFile() && (!pattern || new RegExp(pattern).test(file))) {
+      return fileTransform(filePath);
+    }
+    return [];
+  });
+
+  const resultsNestedArray = await Promise.all(promises);
+  return resultsNestedArray.flat() as T[];
+}
+
+export function findLineNumberInText(
+  content: string,
+  pattern: string,
+): number | null {
+  const lines = content.split(/\r?\n/); // Split lines, handle both Windows and UNIX line endings
+
+  const lineNumber = lines.findIndex(line => line.includes(pattern)) + 1; // +1 because line numbers are 1-based
+  return lineNumber === 0 ? null : lineNumber; // If the package isn't found, return null
+}

packages/utils/src/lib/file-system.unit.test.ts

@@ -7,7 +7,9 @@ import { MEMFS_VOLUME } from '@code-pushup/models/testing';
 import {
   FileResult,
   NoExportError,
+  crawlFileSystem,
   ensureDirectoryExists,
+  findLineNumberInText,
   importEsmModule,
   logMultipleFileResults,
   toUnixPath,
@@ -130,3 +132,86 @@ describe('importEsmModule', () => {
     ).rejects.toThrow(new NoExportError(filepath));
   });
 });
+
+describe('crawlFileSystem', () => {
+  beforeEach(() => {
+    vol.reset();
+    vol.fromJSON(
+      {
+        ['README.md']: '# Markdown',
+        ['src/README.md']: '# Markdown',
+        ['src/index.ts']: 'const var = "markdown";',
+      },
+      outputDir,
+    );
+  });
+
+  it('should list all files in file system', async () => {
+    await expect(
+      crawlFileSystem({
+        directory: outputDir,
+      }),
+    ).resolves.toEqual([
+      expect.stringContaining(join('README.md')),
+      expect.stringContaining(join('README.md')),
+      expect.stringContaining(join('index.ts')),
+    ]);
+  });
+
+  it('should list files matching a pattern', async () => {
+    await expect(
+      crawlFileSystem({
+        directory: outputDir,
+        pattern: /\.md$/,
+      }),
+    ).resolves.toEqual([
+      expect.stringContaining(join('README.md')),
+      expect.stringContaining(join('README.md')),
+    ]);
+  });
+
+  it('should apply sync fileTransform function if given', async () => {
+    await expect(
+      crawlFileSystem({
+        directory: outputDir,
+        pattern: /\.md$/,
+        fileTransform: () => '42',
+      }),
+    ).resolves.toEqual([
+      expect.stringContaining('42'),
+      expect.stringContaining('42'),
+    ]);
+  });
+
+  it('should apply async fileTransform function if given', async () => {
+    await expect(
+      crawlFileSystem({
+        directory: outputDir,
+        pattern: /\.md$/,
+        fileTransform: () => Promise.resolve('42'),
+      }),
+    ).resolves.toEqual([
+      expect.stringContaining('42'),
+      expect.stringContaining('42'),
+    ]);
+  });
+});
+
+describe('findLineNumberInText', () => {
+  it('should return correct line number', () => {
+    expect(
+      findLineNumberInText(
+        `
+    1
+    2 xxx
+    3
+    `,
+        'x',
+      ),
+    ).toBe(3);
+  });
+
+  it('should return null if pattern not in content', () => {
+    expect(findLineNumberInText(``, 'x')).toBeNull();
+  });
+});

packages/utils/src/lib/formatting.ts

@@ -0,0 +1,40 @@
+export function slugify(text: string): string {
+  return text
+    .trim()
+    .toLowerCase()
+    .replace(/\s+|\//g, '-')
+    .replace(/[^a-z0-9-]/g, '');
+}
+
+export function pluralize(text: string): string {
+  if (text.endsWith('y')) {
+    return text.slice(0, -1) + 'ies';
+  }
+  if (text.endsWith('s')) {
+    return `${text}es`;
+  }
+  return `${text}s`;
+}
+
+export function formatBytes(bytes: number, decimals = 2) {
+  if (!+bytes) return '0 B';
+
+  const k = 1024;
+  const dm = decimals < 0 ? 0 : decimals;
+  const sizes = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`;
+}
+
+export function pluralizeToken(token: string, times: number = 0): string {
+  return `${times} ${Math.abs(times) === 1 ? token : pluralize(token)}`;
+}
+
+export function formatDuration(duration: number): string {
+  if (duration < 1000) {
+    return `${duration} ms`;
+  }
+  return `${(duration / 1000).toFixed(2)} s`;
+}

packages/utils/src/lib/formatting.unit.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from 'vitest';
+import {
+  formatBytes,
+  formatDuration,
+  pluralize,
+  pluralizeToken,
+  slugify,
+} from './formatting';
+
+describe('slugify', () => {
+  it.each([
+    ['Largest Contentful Paint', 'largest-contentful-paint'],
+    ['cumulative-layout-shift', 'cumulative-layout-shift'],
+    ['max-lines-200', 'max-lines-200'],
+    ['rxjs/finnish', 'rxjs-finnish'],
+    ['@typescript-eslint/no-explicit-any', 'typescript-eslint-no-explicit-any'],
+    ['Code  PushUp ', 'code-pushup'],
+  ])('should transform "%s" to valid slug "%s"', (text, slug) => {
+    expect(slugify(text)).toBe(slug);
+  });
+});
+
+describe('pluralize', () => {
+  it.each([
+    ['warning', 'warnings'],
+    ['error', 'errors'],
+    ['category', 'categories'],
+    ['status', 'statuses'],
+  ])('should pluralize "%s" as "%s"', (singular, plural) => {
+    expect(pluralize(singular)).toBe(plural);
+  });
+});
+
+describe('formatBytes', () => {
+  it.each([
+    [0, '0 B'],
+    [1000, '1000 B'],
+    [10000, '9.77 kB'],
+    [10000000, '9.54 MB'],
+    [10000000000, '9.31 GB'],
+    [10000000000000, '9.09 TB'],
+    [10000000000000000, '8.88 PB'],
+    [10000000000000000000, '8.67 EB'],
+    [10000000000000000000000, '8.47 ZB'],
+    [10000000000000000000000000, '8.27 YB'],
+    [10000000000000000000000, '8.47 ZB'],
+    [10000000000000000000000, '8.47 ZB'],
+  ])('should log file sizes correctly for %s`', (bytes, displayValue) => {
+    expect(formatBytes(bytes)).toBe(displayValue);
+  });
+
+  it('should log file sizes correctly with correct decimal`', () => {
+    expect(formatBytes(10000, 1)).toBe('9.8 kB');
+  });
+});
+
+describe('pluralizeToken', () => {
+  it.each([
+    [undefined, '0 files'],
+    [-2, '-2 files'],
+    [-1, '-1 file'],
+    [0, '0 files'],
+    [1, '1 file'],
+    [2, '2 files'],
+  ])('should log correct plural for %s`', (times, plural) => {
+    expect(pluralizeToken('file', times)).toBe(plural);
+  });
+});
+
+describe('formatDuration', () => {
+  it.each([
+    [-1, '-1 ms'],
+    [0, '0 ms'],
+    [1, '1 ms'],
+    [2, '2 ms'],
+    [1200, '1.20 s'],
+  ])('should log correct plural for %s`', (ms, displayValue) => {
+    expect(formatDuration(ms)).toBe(displayValue);
+  });
+});

packages/utils/src/lib/promise-result.ts renamed to packages/utils/src/lib/guards.ts

@@ -1,8 +1,5 @@
 import { describe } from 'vitest';
-import {
-  isPromiseFulfilledResult,
-  isPromiseRejectedResult,
-} from './promise-result';
+import { isPromiseFulfilledResult, isPromiseRejectedResult } from './guards';
 
 describe('promise-result', () => {
   it('should get fulfilled result', () => {

packages/utils/src/lib/promise-result.unit.test.ts renamed to packages/utils/src/lib/guards.unit.test.ts

@@ -1,7 +1,4 @@
-import {
-  isPromiseFulfilledResult,
-  isPromiseRejectedResult,
-} from './promise-result';
+import { isPromiseFulfilledResult, isPromiseRejectedResult } from './guards';
 
 export function logMultipleResults<T>(
   results: PromiseSettledResult<T>[],

packages/utils/src/lib/log-results.ts

@@ -1,8 +1,8 @@
 import chalk from 'chalk';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { FileResult } from './file-system';
+import { formatBytes } from './formatting';
 import { logMultipleResults, logPromiseResults } from './log-results';
-import { formatBytes } from './report';
 
 const succeededCallback = (result: PromiseFulfilledResult<FileResult>) => {
   const [fileName, size] = result.value;