feat(vitest): add github actions reporter (#5093)

Author: hi-ogawa

docs/guide/reporters.md

@@ -445,6 +445,16 @@ export default defineConfig({
 ```
 :::
 
+### Github Actions Reporter <Badge type="info">1.3.0+</Badge>
+
+Output [workflow commands](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)
+to provide annotations for test failures. This reporter is automatically enabled when `process.env.GITHUB_ACTIONS === 'true'`, thus it doesn't require any configuration.
+
+<img alt="Github Actions" img-dark src="https://github.com/vitest-dev/vitest/assets/4232207/336cddc2-df6b-4b8a-8e72-4d00010e37f5">
+<img alt="Github Actions" img-light src="https://github.com/vitest-dev/vitest/assets/4232207/ce8447c1-0eab-4fe1-abef-d0d322290dca">
+
+
+
 ## Custom Reporters
 
 You can use third-party custom reporters installed from NPM by specifying their package name in the reporters' option:

packages/vitest/src/node/config.ts

@@ -422,6 +422,10 @@ export function resolveConfig(
   if (!resolved.reporters.length)
     resolved.reporters.push(['default', {}])
 
+  // automatically enable github-actions reporter
+  if (process.env.GITHUB_ACTIONS === 'true' && !resolved.reporters.some(v => Array.isArray(v) && v[0] === 'github-actions'))
+    resolved.reporters.push(['github-actions', {}])
+
   if (resolved.changed)
     resolved.passWithNoTests ??= true
 

packages/vitest/src/node/error.ts

@@ -22,7 +22,11 @@ interface PrintErrorOptions {
   showCodeFrame?: boolean
 }
 
-export async function printError(error: unknown, project: WorkspaceProject | undefined, options: PrintErrorOptions) {
+interface PrintErrorResult {
+  nearest?: ParsedStack
+}
+
+export async function printError(error: unknown, project: WorkspaceProject | undefined, options: PrintErrorOptions): Promise<PrintErrorResult | undefined> {
   const { showCodeFrame = true, fullStack = false, type } = options
   const logger = options.logger
   let e = error as ErrorWithDiff
@@ -43,8 +47,10 @@ export async function printError(error: unknown, project: WorkspaceProject | und
   }
 
   // Error may have occured even before the configuration was resolved
-  if (!project)
-    return printErrorMessage(e, logger)
+  if (!project) {
+    printErrorMessage(e, logger)
+    return
+  }
 
   const parserOptions: StackTraceParserOptions = {
     // only browser stack traces require remapping
@@ -85,7 +91,7 @@ export async function printError(error: unknown, project: WorkspaceProject | und
     logger.error(c.yellow(e.frame))
   }
   else {
-    printStack(project, stacks, nearest, errorProperties, (s) => {
+    printStack(logger, project, stacks, nearest, errorProperties, (s) => {
       if (showCodeFrame && s === nearest && nearest) {
         const sourceCode = readFileSync(nearest.file, 'utf-8')
         logger.error(generateCodeFrame(sourceCode.length > 100_000 ? sourceCode : logger.highlight(nearest.file, sourceCode), 4, s))
@@ -116,6 +122,8 @@ export async function printError(error: unknown, project: WorkspaceProject | und
   }
 
   handleImportOutsideModuleError(e.stack || e.stackStr || '', logger)
+
+  return { nearest }
 }
 
 function printErrorType(type: string, ctx: Vitest) {
@@ -229,14 +237,13 @@ function printErrorMessage(error: ErrorWithDiff, logger: Logger) {
 }
 
 function printStack(
+  logger: Logger,
   project: WorkspaceProject,
   stack: ParsedStack[],
   highlight: ParsedStack | undefined,
   errorProperties: Record<string, unknown>,
   onStack?: ((stack: ParsedStack) => void),
 ) {
-  const logger = project.ctx.logger
-
   for (const frame of stack) {
     const color = frame === highlight ? c.cyan : c.gray
     const path = relative(project.config.root, frame.file)

packages/vitest/src/node/logger.ts

@@ -82,10 +82,10 @@ export class Logger {
     this.console.log(`${CURSOR_TO_START}${ERASE_DOWN}${log}`)
   }
 
-  printError(err: unknown, options: ErrorOptions = {}) {
+  async printError(err: unknown, options: ErrorOptions = {}) {
     const { fullStack = false, type } = options
     const project = options.project ?? this.ctx.getCoreWorkspaceProject() ?? this.ctx.projects[0]
-    return printError(err, project, {
+    await printError(err, project, {
       fullStack,
       type,
       showCodeFrame: true,

packages/vitest/src/node/reporters/github-actions.ts

@@ -0,0 +1,116 @@
+import { Console } from 'node:console'
+import { Writable } from 'node:stream'
+import { getTasks } from '@vitest/runner/utils'
+import stripAnsi from 'strip-ansi'
+import type { File, Reporter, Vitest } from '../../types'
+import { getFullName } from '../../utils'
+import { printError } from '../error'
+import { Logger } from '../logger'
+import type { WorkspaceProject } from '../workspace'
+
+export class GithubActionsReporter implements Reporter {
+  ctx: Vitest = undefined!
+
+  onInit(ctx: Vitest) {
+    this.ctx = ctx
+  }
+
+  async onFinished(files: File[] = [], errors: unknown[] = []) {
+    // collect all errors and associate them with projects
+    const projectErrors = new Array<{ project: WorkspaceProject; title: string; error: unknown }>()
+    for (const error of errors) {
+      projectErrors.push({
+        project: this.ctx.getCoreWorkspaceProject(),
+        title: 'Unhandled error',
+        error,
+      })
+    }
+    for (const file of files) {
+      const tasks = getTasks(file)
+      const project = this.ctx.getProjectByTaskId(file.id)
+      for (const task of tasks) {
+        if (task.result?.state !== 'fail')
+          continue
+
+        const title = getFullName(task, ' > ')
+        for (const error of task.result?.errors ?? []) {
+          projectErrors.push({
+            project,
+            title,
+            error,
+          })
+        }
+      }
+    }
+
+    // format errors via `printError`
+    for (const { project, title, error } of projectErrors) {
+      const result = await printErrorWrapper(error, this.ctx, project)
+      const stack = result?.nearest
+      if (!stack)
+        continue
+      const formatted = formatMessage({
+        command: 'error',
+        properties: {
+          file: stack.file,
+          title,
+          line: String(stack.line),
+          column: String(stack.column),
+        },
+        message: stripAnsi(result.output),
+      })
+      this.ctx.logger.log(`\n${formatted}`)
+    }
+  }
+}
+
+// use Logger with custom Console to extract messgage from `processError` util
+// TODO: maybe refactor `processError` to require single function `(message: string) => void` instead of full Logger?
+async function printErrorWrapper(error: unknown, ctx: Vitest, project: WorkspaceProject) {
+  let output = ''
+  const writable = new Writable({
+    write(chunk, _encoding, callback) {
+      output += String(chunk)
+      callback()
+    },
+  })
+  const result = await printError(error, project, {
+    showCodeFrame: false,
+    logger: new Logger(ctx, new Console(writable, writable)),
+  })
+  return { nearest: result?.nearest, output }
+}
+
+// workflow command formatting based on
+// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
+// https://github.com/actions/toolkit/blob/f1d9b4b985e6f0f728b4b766db73498403fd5ca3/packages/core/src/command.ts#L80-L85
+function formatMessage({
+  command,
+  properties,
+  message,
+}: {
+  command: string
+  properties: Record<string, string>
+  message: string
+}): string {
+  let result = `::${command}`
+  Object.entries(properties).forEach(([k, v], i) => {
+    result += i === 0 ? ' ' : ','
+    result += `${k}=${escapeProperty(v)}`
+  })
+  result += `::${escapeData(message)}`
+  return result
+}
+
+function escapeData(s: string): string {
+  return s.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
+}
+
+function escapeProperty(s: string): string {
+  return s
+    .replace(/%/g, '%25')
+    .replace(/\r/g, '%0D')
+    .replace(/\n/g, '%0A')
+    .replace(/:/g, '%3A')
+    .replace(/,/g, '%2C')
+}

packages/vitest/src/node/reporters/index.ts

@@ -8,6 +8,7 @@ import { TapReporter } from './tap'
 import { type JUnitOptions, JUnitReporter } from './junit'
 import { TapFlatReporter } from './tap-flat'
 import { HangingProcessReporter } from './hanging-process'
+import { GithubActionsReporter } from './github-actions'
 import type { BaseReporter } from './base'
 
 export {
@@ -20,6 +21,7 @@ export {
   JUnitReporter,
   TapFlatReporter,
   HangingProcessReporter,
+  GithubActionsReporter,
 }
 export type { BaseReporter, Reporter }
 
@@ -35,6 +37,7 @@ export const ReportersMap = {
   'tap-flat': TapFlatReporter,
   'junit': JUnitReporter,
   'hanging-process': HangingProcessReporter,
+  'github-actions': GithubActionsReporter,
 }
 
 export type BuiltinReporters = keyof typeof ReportersMap

test/reporters/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "private": true,
   "scripts": {
-    "test": "NO_COLOR=1 vitest run"
+    "test": "NO_COLOR=1 GITHUB_ACTIONS=false vitest run"
   },
   "devDependencies": {
     "flatted": "^3.2.9",

test/reporters/tests/github-actions.test.ts

@@ -0,0 +1,18 @@
+import { resolve } from 'pathe'
+import { expect, test } from 'vitest'
+import { runVitest } from '../../test-utils'
+import { GithubActionsReporter } from '../../../packages/vitest/src/node/reporters'
+
+test(GithubActionsReporter, async () => {
+  let { stdout, stderr } = await runVitest(
+    { reporters: new GithubActionsReporter(), root: './fixtures' },
+    ['some-failing.test.ts'],
+  )
+  stdout = stdout.replace(resolve(__dirname, '..').replace(/:/g, '%3A'), '__TEST_DIR__')
+  expect(stdout).toMatchInlineSnapshot(`
+    "
+    ::error file=__TEST_DIR__/fixtures/some-failing.test.ts,title=some-failing.test.ts > 3 + 3 = 7,line=8,column=17::AssertionError: expected 6 to be 7 // Object.is equality%0A%0A- Expected%0A+ Received%0A%0A- 7%0A+ 6%0A%0A ❯ some-failing.test.ts:8:17%0A%0A
+    "
+  `)
+  expect(stderr).toBe('')
+})

test/ui/package.json

@@ -3,7 +3,7 @@
   "type": "module",
   "private": true,
   "scripts": {
-    "test-e2e": "playwright test",
+    "test-e2e": "GITHUB_ACTIONS=false playwright test",
     "test-fixtures": "vitest"
   },
   "devDependencies": {