fix(benchmark): run benchmark suites sequentially (#5444)

Author: hi-ogawa

.gitignore

@@ -15,7 +15,7 @@ dist
 ltex*
 .DS_Store
 bench/test/*/*/
-**/benchmark/bench.json
+**/bench.json
 **/browser/browser.json
 cypress/videos
 cypress/downloads

packages/vitest/src/node/config.ts

@@ -119,7 +119,8 @@ export function resolveConfig(
   if (resolved.minWorkers)
     resolved.minWorkers = Number(resolved.minWorkers)
 
-  resolved.fileParallelism ??= true
+  // run benchmark sequentially by default
+  resolved.fileParallelism ??= mode !== 'benchmark'
 
   if (!resolved.fileParallelism) {
     // ignore user config, parallelism cannot be implemented without limiting workers

packages/vitest/src/runtime/runners/benchmark.ts

@@ -35,8 +35,9 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) {
       benchmarkSuiteGroup.push(task)
   }
 
-  if (benchmarkSuiteGroup.length)
-    await Promise.all(benchmarkSuiteGroup.map(subSuite => runBenchmarkSuite(subSuite, runner)))
+  // run sub suites sequentially
+  for (const subSuite of benchmarkSuiteGroup)
+    await runBenchmarkSuite(subSuite, runner)
 
   if (benchmarkGroup.length) {
     const defer = createDefer()

test/benchmark/test/base.bench.ts renamed to test/benchmark/fixtures/basic/base.bench.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({})

test/benchmark/test/mode.bench.ts renamed to test/benchmark/fixtures/basic/mode.bench.ts

@@ -0,0 +1,26 @@
+import { bench, describe } from "vitest"
+import { appendLog, benchOptions, sleepBench } from "./helper";
+
+bench("B1", async () => {
+  await appendLog("F1 / B1")
+  await sleepBench();
+}, benchOptions)
+
+describe("S1", () => {
+  bench("B1", async () => {
+    await appendLog("F1 / S1 / B1")
+    await sleepBench();
+  }, benchOptions)
+
+  bench("B2", async () => {
+    await appendLog("F1 / S1 / B2")
+    await sleepBench();
+  }, benchOptions)
+})
+
+describe("S2", () => {
+  bench("B1", async () => {
+    await appendLog("F1 / S2 / B1")
+    await sleepBench();
+  }, benchOptions)
+})

test/benchmark/test/only.bench.ts renamed to test/benchmark/fixtures/basic/only.bench.ts

@@ -0,0 +1,9 @@
+import { bench, describe } from "vitest"
+import { appendLog, benchOptions, sleepBench } from "./helper";
+
+describe("S1", () => {
+  bench("B1", async () => {
+    await appendLog("F2 / S1 / B1")
+    await sleepBench();
+  }, benchOptions)
+})

test/benchmark/fixtures/basic/vitest.config.ts

@@ -0,0 +1,19 @@
+import fs from "node:fs";
+
+const SLEEP_BENCH_MS = Number(process.env["SLEEP_BENCH_MS"] || 10);
+const BENCH_ITERATIONS = Number(process.env["BENCH_ITERATIONS"] || 3);
+
+export const sleepBench = () => new Promise(resolve => setTimeout(resolve, SLEEP_BENCH_MS))
+
+export const testLogFile = new URL("./test.log", import.meta.url);
+
+export async function appendLog(data: string) {
+  await fs.promises.appendFile(testLogFile, data + "\n");
+}
+
+export const benchOptions = {
+  time: 0,
+  iterations: BENCH_ITERATIONS,
+  warmupIterations: 0,
+  warmupTime: 0,
+}

test/benchmark/fixtures/sequential/f1.bench.ts

@@ -0,0 +1,6 @@
+import fs from "node:fs";
+import { testLogFile } from "./helper";
+
+export default async function setup() {
+  await fs.promises.rm(testLogFile, { force: true });
+}

test/benchmark/fixtures/sequential/f2.bench.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "vitest/config"
+
+// to see the difference better, increase sleep time and iterations e.g. by
+// SLEEP_BENCH_MS=100 pnpm -C test/benchmark test bench -- --root fixtures/sequential --fileParallelism
+
+export default defineConfig({
+  test: {
+    globalSetup: ["./setup.ts"]
+  }
+});

test/benchmark/fixtures/sequential/helper.ts

@@ -1,11 +1,9 @@
 {
-  "name": "@vitest/benchmark",
+  "name": "@vitest/benchmark-sequential",
   "type": "module",
   "private": true,
   "scripts": {
-    "test": "node --test specs/* && echo '1'",
-    "bench:json": "vitest bench --reporter=json",
-    "bench": "vitest bench"
+    "test": "vitest"
   },
   "devDependencies": {
     "vitest": "workspace:*"

test/benchmark/fixtures/sequential/setup.ts

@@ -0,0 +1,20 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`sequential 1`] = `
+"F1 / S1 / B1
+F1 / S1 / B1
+F1 / S1 / B1
+F1 / S1 / B2
+F1 / S1 / B2
+F1 / S1 / B2
+F1 / S2 / B1
+F1 / S2 / B1
+F1 / S2 / B1
+F1 / B1
+F1 / B1
+F1 / B1
+F2 / S1 / B1
+F2 / S1 / B1
+F2 / S1 / B1
+"
+`;

test/benchmark/fixtures/sequential/vitest.config.ts

@@ -0,0 +1,38 @@
+import fs from 'node:fs'
+import { expect, it } from 'vitest'
+import * as pathe from 'pathe'
+import { runVitest } from '../../test-utils'
+
+it('basic', { timeout: 60_000 }, async () => {
+  const root = pathe.join(import.meta.dirname, '../fixtures/basic')
+  const benchFile = pathe.join(root, 'bench.json')
+  fs.rmSync(benchFile, { force: true })
+
+  await runVitest({
+    root,
+    allowOnly: true,
+    benchmark: {
+      reporters: 'json',
+      outputFile: 'bench.json',
+    },
+  }, [], 'benchmark')
+
+  const benchResult = await fs.promises.readFile(benchFile, 'utf-8')
+  const resultJson = JSON.parse(benchResult)
+
+  expect(Object.keys(resultJson.testResults)).toEqual(
+    expect.arrayContaining([
+      'sort',
+      'timeout',
+      'a0',
+      'c1',
+      'a2',
+      'b3',
+      'b4',
+    ]),
+  )
+
+  const skipped = ['skip', 's0', 's1', 's2', 's3', 'sb4', 's4', 'unimplemented suite', 'unimplemented test']
+  for (const b of skipped)
+    expect(benchResult).not.toContain(b)
+})

test/benchmark/package.json

@@ -0,0 +1,11 @@
+import fs from 'node:fs'
+import { expect, it } from 'vitest'
+import * as pathe from 'pathe'
+import { runVitest } from '../../test-utils'
+
+it('sequential', async () => {
+  const root = pathe.join(import.meta.dirname, '../fixtures/sequential')
+  await runVitest({ root }, [], 'benchmark')
+  const testLog = await fs.promises.readFile(pathe.join(root, 'test.log'), 'utf-8')
+  expect(testLog).toMatchSnapshot()
+})

test/benchmark/specs/runner.test.mjs

@@ -1,25 +1,3 @@
 import { defineConfig } from 'vitest/config'
 
-function noop() {}
-
-export default defineConfig({
-  test: {
-    update: false,
-    allowOnly: true,
-    benchmark: {
-      outputFile: './bench.json',
-      reporters: ['json', {
-        onInit: noop,
-        onPathsCollected: noop,
-        onCollected: noop,
-        onFinished: noop,
-        onTaskUpdate: noop,
-        onTestRemoved: noop,
-        onWatcherStart: noop,
-        onWatcherRerun: noop,
-        onServerRestart: noop,
-        onUserConsoleLog: noop,
-      }],
-    },
-  },
-})
+export default defineConfig({})