fix(vitest): show all failed tests when rerunning a test (#6022)

Author: sheremet-va

packages/vitest/src/api/setup.ts

@@ -201,7 +201,7 @@ export class WebSocketReporter implements Reporter {
     })
   }
 
-  onFinished(files?: File[], errors?: unknown[]) {
+  onFinished(files: File[], errors: unknown[]) {
     this.clients.forEach((client) => {
       client.onFinished?.(files, errors)?.catch?.(noop)
     })

packages/vitest/src/node/pools/typecheck.ts

@@ -40,7 +40,7 @@ export function createTypecheckPool(ctx: Vitest): ProcessPool {
 
     // triggered by TSC watcher, not Vitest watcher, so we need to emulate what Vitest does in this case
     if (ctx.config.watch && !ctx.runningPromise) {
-      await ctx.report('onFinished', files)
+      await ctx.report('onFinished', files, [])
       await ctx.report('onWatcherStart', files, [
         ...(project.config.typecheck.ignoreSourceErrors ? [] : sourceErrors),
         ...ctx.state.getUnhandledErrors(),

packages/vitest/src/node/reporters/base.ts

@@ -14,6 +14,7 @@ import {
   getFullName,
   getSafeTimers,
   getSuites,
+  getTestName,
   getTests,
   hasFailed,
   hasFailedSnapshot,
@@ -33,8 +34,8 @@ import {
   formatTimeString,
   getStateString,
   getStateSymbol,
-  pointer,
   renderSnapshotSummary,
+  taskFail,
 } from './renderers/utils'
 
 const BADGE_PADDING = '       '
@@ -63,6 +64,7 @@ export abstract class BaseReporter implements Reporter {
   start = 0
   end = 0
   watchFilters?: string[]
+  failedUnwatchedFiles: Task[] = []
   isTTY: boolean
   ctx: Vitest = undefined!
 
@@ -115,59 +117,65 @@ export abstract class BaseReporter implements Reporter {
     if (this.isTTY) {
       return
     }
-    const logger = this.ctx.logger
     for (const pack of packs) {
       const task = this.ctx.state.idMap.get(pack[0])
-      if (
-        task
-        && 'filepath' in task
-        && task.result?.state
-        && task.result?.state !== 'run'
-      ) {
-        const tests = getTests(task)
-        const failed = tests.filter(t => t.result?.state === 'fail')
-        const skipped = tests.filter(
-          t => t.mode === 'skip' || t.mode === 'todo',
-        )
-        let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`)
-        if (failed.length) {
-          state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}`
-        }
-        if (skipped.length) {
-          state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}`
-        }
-        let suffix = c.dim(' (') + state + c.dim(')')
-        if (task.result.duration) {
-          const color
-            = task.result.duration > this.ctx.config.slowTestThreshold
-              ? c.yellow
-              : c.gray
-          suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`)
-        }
-        if (this.ctx.config.logHeapUsage && task.result.heap != null) {
-          suffix += c.magenta(
-            ` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`,
-          )
-        }
-
-        let title = ` ${getStateSymbol(task)} `
-        if (task.projectName) {
-          title += formatProjectName(task.projectName)
-        }
-        title += `${task.name} ${suffix}`
-        logger.log(title)
-
-        // print short errors, full errors will be at the end in summary
-        for (const test of failed) {
-          logger.log(c.red(`   ${pointer} ${getFullName(test, c.dim(' > '))}`))
-          test.result?.errors?.forEach((e) => {
-            logger.log(c.red(`     ${F_RIGHT} ${(e as any)?.message}`))
-          })
-        }
+      if (task) {
+        this.printTask(task)
       }
     }
   }
 
+  protected printTask(task: Task) {
+    if (
+      !('filepath' in task)
+      || !task.result?.state
+      || task.result?.state === 'run') {
+      return
+    }
+    const logger = this.ctx.logger
+
+    const tests = getTests(task)
+    const failed = tests.filter(t => t.result?.state === 'fail')
+    const skipped = tests.filter(
+      t => t.mode === 'skip' || t.mode === 'todo',
+    )
+    let state = c.dim(`${tests.length} test${tests.length > 1 ? 's' : ''}`)
+    if (failed.length) {
+      state += ` ${c.dim('|')} ${c.red(`${failed.length} failed`)}`
+    }
+    if (skipped.length) {
+      state += ` ${c.dim('|')} ${c.yellow(`${skipped.length} skipped`)}`
+    }
+    let suffix = c.dim(' (') + state + c.dim(')')
+    if (task.result.duration) {
+      const color
+        = task.result.duration > this.ctx.config.slowTestThreshold
+          ? c.yellow
+          : c.gray
+      suffix += color(` ${Math.round(task.result.duration)}${c.dim('ms')}`)
+    }
+    if (this.ctx.config.logHeapUsage && task.result.heap != null) {
+      suffix += c.magenta(
+        ` ${Math.floor(task.result.heap / 1024 / 1024)} MB heap used`,
+      )
+    }
+
+    let title = ` ${getStateSymbol(task)} `
+    if (task.projectName) {
+      title += formatProjectName(task.projectName)
+    }
+    title += `${task.name} ${suffix}`
+    logger.log(title)
+
+    // print short errors, full errors will be at the end in summary
+    for (const test of failed) {
+      logger.log(c.red(`   ${taskFail} ${getTestName(test, c.dim(' > '))}`))
+      test.result?.errors?.forEach((e) => {
+        logger.log(c.red(`     ${F_RIGHT} ${(e as any)?.message}`))
+      })
+    }
+  }
+
   onWatcherStart(
     files = this.ctx.state.getFiles(),
     errors = this.ctx.state.getUnhandledErrors(),
@@ -233,6 +241,9 @@ export abstract class BaseReporter implements Reporter {
   onWatcherRerun(files: string[], trigger?: string) {
     this.resetLastRunLog()
     this.watchFilters = files
+    this.failedUnwatchedFiles = this.ctx.state.getFiles().filter((file) => {
+      return !files.includes(file.filepath) && hasFailed(file)
+    })
 
     files.forEach((filepath) => {
       let reruns = this._filesInWatchMode.get(filepath) ?? 0
@@ -274,6 +285,12 @@ export abstract class BaseReporter implements Reporter {
       )
     }
 
+    if (!this.isTTY) {
+      for (const task of this.failedUnwatchedFiles) {
+        this.printTask(task)
+      }
+    }
+
     this._timeStart = new Date()
     this.start = performance.now()
   }
@@ -375,7 +392,11 @@ export abstract class BaseReporter implements Reporter {
   }
 
   reportTestSummary(files: File[], errors: unknown[]) {
-    const tests = getTests(files)
+    const affectedFiles = [
+      ...this.failedUnwatchedFiles,
+      ...files,
+    ]
+    const tests = getTests(affectedFiles)
     const logger = this.ctx.logger
 
     const executionTime = this.end - this.start
@@ -437,7 +458,7 @@ export abstract class BaseReporter implements Reporter {
       }
     }
 
-    logger.log(padTitle('Test Files'), getStateString(files))
+    logger.log(padTitle('Test Files'), getStateString(affectedFiles))
     logger.log(padTitle('Tests'), getStateString(tests))
     if (this.ctx.projects.some(c => c.config.typecheck.enabled)) {
       const failed = tests.filter(

packages/vitest/src/node/reporters/basic.ts

@@ -2,7 +2,10 @@ import type { File } from '../../types/tasks'
 import { BaseReporter } from './base'
 
 export class BasicReporter extends BaseReporter {
-  isTTY = false
+  constructor() {
+    super()
+    this.isTTY = false
+  }
 
   reportSummary(files: File[], errors: unknown[]) {
     // non-tty mode doesn't add a new line

packages/vitest/src/node/reporters/default.ts

@@ -56,6 +56,16 @@ export class DefaultReporter extends BaseReporter {
     files = this.ctx.state.getFiles(),
     errors = this.ctx.state.getUnhandledErrors(),
   ) {
+    // print failed tests without their errors to keep track of previously failed tests
+    // this can happen if there are multiple test errors, and user changed a file
+    // that triggered a rerun of unrelated tests - in that case they want to see
+    // the error for the test they are currently working on, but still keep track of
+    // the other failed tests
+    this.renderer?.update([
+      ...this.failedUnwatchedFiles,
+      ...files,
+    ])
+
     this.stopListRender()
     this.ctx.logger.log()
     super.onFinished(files, errors)

packages/vitest/src/node/reporters/renderers/utils.ts

@@ -19,6 +19,12 @@ export const hookSpinnerMap = new WeakMap<Task, Map<string, () => string>>()
 export const pointer = c.yellow(F_POINTER)
 export const skipped = c.dim(c.gray(F_DOWN))
 
+export const benchmarkPass = c.green(F_DOT)
+export const testPass = c.green(F_CHECK)
+export const taskFail = c.red(F_CROSS)
+export const suiteFail = c.red(F_POINTER)
+export const pending = c.gray('·')
+
 export function getCols(delta = 0) {
   let length = process.stdout?.columns
   if (!length || Number.isNaN(length)) {
@@ -154,10 +160,9 @@ export function getStateSymbol(task: Task) {
   }
 
   if (!task.result) {
-    return c.gray('·')
+    return pending
   }
 
-  // pending
   if (task.result.state === 'run') {
     if (task.type === 'suite') {
       return pointer
@@ -171,11 +176,11 @@ export function getStateSymbol(task: Task) {
   }
 
   if (task.result.state === 'pass') {
-    return task.meta?.benchmark ? c.green(F_DOT) : c.green(F_CHECK)
+    return task.meta?.benchmark ? benchmarkPass : testPass
   }
 
   if (task.result.state === 'fail') {
-    return task.type === 'suite' ? pointer : c.red(F_CROSS)
+    return task.type === 'suite' ? suiteFail : taskFail
   }
 
   return ' '

packages/vitest/src/types/reporter.ts

@@ -8,8 +8,8 @@ export interface Reporter {
   onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable<void>
   onCollected?: (files?: File[]) => Awaitable<void>
   onFinished?: (
-    files?: File[],
-    errors?: unknown[],
+    files: File[],
+    errors: unknown[],
     coverage?: unknown
   ) => Awaitable<void>
   onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable<void>

test/core/vite.config.ts

@@ -50,7 +50,6 @@ export default defineConfig({
     port: 3022,
   },
   test: {
-    reporters: ['dot'],
     api: {
       port: 3023,
     },

test/reporters/tests/merge-reports.test.ts

@@ -89,13 +89,13 @@ test('merge reports', async () => {
     test 1-2
 
      ❯ first.test.ts  (2 tests | 1 failed) <time>
-       ❯ first.test.ts > test 1-2
+       × test 1-2
          → expected 1 to be 2 // Object.is equality
     stdout | second.test.ts > test 2-1
     test 2-1
 
      ❯ second.test.ts  (3 tests | 1 failed) <time>
-       ❯ second.test.ts > test 2-1
+       × test 2-1
          → expected 1 to be 2 // Object.is equality
 
      Test Files  2 failed (2)

test/watch/fixtures/single-failed/basic.test.ts

@@ -0,0 +1,6 @@
+import { expect, it } from 'vitest';
+
+it('works correctly', () => {
+  console.log('log basic')
+  expect(1).toBe(1)
+})

test/watch/fixtures/single-failed/failed.test.ts

@@ -0,0 +1,6 @@
+import { it } from 'vitest';
+
+it('fails', () => {
+  console.log('log fail')
+  throw new Error('failed')
+})

test/watch/fixtures/single-failed/vitest.config.ts

@@ -0,0 +1,11 @@
+import { defineConfig } from 'vitest/config'
+
+// Patch stdin on the process so that we can fake it to seem like a real interactive terminal and pass the TTY checks
+process.stdin.isTTY = true
+process.stdin.setRawMode = () => process.stdin
+
+export default defineConfig({
+  test: {
+    watch: true,
+  },
+})

test/watch/fixtures/vitest.config.ts

@@ -1,4 +1,4 @@
-import { defineConfig } from 'vitest/config'
+import { defaultExclude, defineConfig } from 'vitest/config'
 
 // Patch stdin on the process so that we can fake it to seem like a real interactive terminal and pass the TTY checks
 process.stdin.isTTY = true
@@ -7,6 +7,10 @@ process.stdin.setRawMode = () => process.stdin
 export default defineConfig({
   test: {
     watch: true,
+    exclude: [
+      ...defaultExclude,
+      '**/single-failed/**',
+    ],
 
     // This configuration is edited by tests
     reporters: 'verbose',