feat(vitest)!: add "vitest list" API to print collected tests without running them (#6013)

Author: sheremet-va

docs/advanced/pool.md

@@ -45,6 +45,7 @@ import { ProcessPool, WorkspaceProject } from 'vitest/node'
 export interface ProcessPool {
   name: string
   runTests: (files: [project: WorkspaceProject, testFile: string][], invalidates?: string[]) => Promise<void>
+  collectTests: (files: [project: WorkspaceProject, testFile: string][], invalidates?: string[]) => Promise<void>
   close?: () => Promise<void>
 }
 ```
@@ -57,6 +58,8 @@ Vitest will wait until `runTests` is executed before finishing a run (i.e., it w
 
 If you are using a custom pool, you will have to provide test files and their results yourself - you can reference [`vitest.state`](https://github.com/vitest-dev/vitest/blob/main/packages/vitest/src/node/state.ts) for that (most important are `collectFiles` and `updateTasks`). Vitest uses `startTests` function from `@vitest/runner` package to do that.
 
+Vitest will call `collectTests` if `vitest.collect` is called or `vitest list` is invoked via a CLI command. It works the same way as `runTests`, but you don't have to run test callbacks, only report their tasks by calling `vitest.state.collectFiles(files)`.
+
 To communicate between different processes, you can create methods object using `createMethodsRPC` from `vitest/node`, and use any form of communication that you prefer. For example, to use WebSockets with `birpc` you can write something like this:
 
 ```ts

docs/guide/cli.md

@@ -55,6 +55,36 @@ export default {
 
 Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experimental) tests, which compare performance results.
 
+### `vitest init`
+
+`vitest init <name>` can be used to setup project configuration. At the moment, it only supports [`browser`](/guide/browser) value:
+
+```bash
+vitest init browser
+```
+
+### `vitest list`
+
+`vitest list` command inherits all `vitest` options to print the list of all matching tests. This command ignores `reporters` option. By default, it will print the names of all tests that matched the file filter and name pattern:
+
+```shell
+vitest list filename.spec.ts -t="some-test"
+```
+
+```txt
+describe > some-test
+describe > some-test > test 1
+describe > some-test > test 2
+```
+
+You can pass down `--json` flag to print tests in JSON format or save it in a separate file:
+
+```bash
+vitest list filename.spec.ts -t="some-test" --json=./file.json
+```
+
+If `--json` flag doesn't receive a value, it will output the JSON into stdout.
+
 ## Options
 
 <!--@include: ./cli-table.md-->

packages/browser/src/client/tester/tester.ts

@@ -1,4 +1,4 @@
-import { SpyModule, setupCommonEnv, startTests } from 'vitest/browser'
+import { SpyModule, collectTests, setupCommonEnv, startTests } from 'vitest/browser'
 import { getBrowserState, getConfig, getWorkerState } from '../utils'
 import { channel, client, onCancel } from '../client'
 import { setupDialogsSpy } from './dialog'
@@ -65,8 +65,6 @@ async function prepareTestEnvironment(files: string[]) {
     runner,
     config,
     state,
-    setupCommonEnv,
-    startTests,
   }
 }
 
@@ -78,7 +76,7 @@ function done(files: string[]) {
   })
 }
 
-async function runTests(files: string[]) {
+async function executeTests(method: 'run' | 'collect', files: string[]) {
   await client.waitForConnection()
 
   debug('client is connected to ws server')
@@ -107,7 +105,7 @@ async function runTests(files: string[]) {
 
   debug('runner resolved successfully')
 
-  const { config, runner, state, setupCommonEnv, startTests } = preparedData
+  const { config, runner, state } = preparedData
 
   state.durations.prepare = performance.now() - state.durations.prepare
 
@@ -116,7 +114,12 @@ async function runTests(files: string[]) {
   try {
     await setupCommonEnv(config)
     for (const file of files) {
-      await startTests([file], runner)
+      if (method === 'run') {
+        await startTests([file], runner)
+      }
+      else {
+        await collectTests([file], runner)
+      }
     }
   }
   finally {
@@ -127,4 +130,6 @@ async function runTests(files: string[]) {
 }
 
 // @ts-expect-error untyped global for internal use
-window.__vitest_browser_runner__.runTests = runTests
+window.__vitest_browser_runner__.runTests = files => executeTests('run', files)
+// @ts-expect-error untyped global for internal use
+window.__vitest_browser_runner__.collectTests = files => executeTests('collect', files)

packages/browser/src/node/pool.ts

@@ -10,15 +10,16 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
   const providers = new Set<BrowserProvider>()
 
   const waitForTests = async (
+    method: 'run' | 'collect',
     contextId: string,
     project: WorkspaceProject,
     files: string[],
   ) => {
-    const context = project.browser!.state.createAsyncContext(contextId, files)
+    const context = project.browser!.state.createAsyncContext(method, contextId, files)
     return await context
   }
 
-  const runTests = async (project: WorkspaceProject, files: string[]) => {
+  const executeTests = async (method: 'run' | 'collect', project: WorkspaceProject, files: string[]) => {
     ctx.state.clearFiles(project, files)
     const browser = project.browser!
 
@@ -67,13 +68,13 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
           contextId,
           [...files.map(f => relative(project.config.root, f))].join(', '),
         )
-        const promise = waitForTests(contextId, project, files)
+        const promise = waitForTests(method, contextId, project, files)
         promises.push(promise)
         orchestrator.createTesters(files)
       }
       else {
         const contextId = crypto.randomUUID()
-        const waitPromise = waitForTests(contextId, project, files)
+        const waitPromise = waitForTests(method, contextId, project, files)
         debug?.(
           'Opening a new context %s for files: %s',
           contextId,
@@ -91,7 +92,7 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
     await Promise.all(promises)
   }
 
-  const runWorkspaceTests = async (specs: [WorkspaceProject, string][]) => {
+  const runWorkspaceTests = async (method: 'run' | 'collect', specs: [WorkspaceProject, string][]) => {
     const groupedFiles = new Map<WorkspaceProject, string[]>()
     for (const [project, file] of specs) {
       const files = groupedFiles.get(project) || []
@@ -110,7 +111,7 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
         break
       }
 
-      await runTests(project, files)
+      await executeTests(method, project, files)
     }
   }
 
@@ -140,6 +141,7 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
       await Promise.all([...providers].map(provider => provider.close()))
       providers.clear()
     },
-    runTests: runWorkspaceTests,
+    runTests: files => runWorkspaceTests('run', files),
+    collectTests: files => runWorkspaceTests('collect', files),
   }
 }

packages/browser/src/node/serverTester.ts

@@ -29,7 +29,9 @@ export async function resolveTester(
       ? '__vitest_browser_runner__.files'
       : JSON.stringify([testFile])
   const iframeId = JSON.stringify(testFile)
-  const files = state.getContext(contextId)?.files ?? []
+  const context = state.getContext(contextId)
+  const files = context?.files ?? []
+  const method = context?.method ?? 'run'
 
   const injectorJs = typeof server.injectorJs === 'string'
     ? server.injectorJs
@@ -74,7 +76,7 @@ export async function resolveTester(
       `<script type="module">
 __vitest_browser_runner__.runningFiles = ${tests}
 __vitest_browser_runner__.iframeId = ${iframeId}
-__vitest_browser_runner__.runTests(__vitest_browser_runner__.runningFiles)
+__vitest_browser_runner__.${method === 'run' ? 'runTests' : 'collectTests'}(__vitest_browser_runner__.runningFiles)
 </script>`,
   })
 }

packages/browser/src/node/state.ts

@@ -14,10 +14,11 @@ export class BrowserServerState implements IBrowserServerState {
     return this.contexts.get(contextId)
   }
 
-  createAsyncContext(contextId: string, files: string[]): Promise<void> {
+  createAsyncContext(method: 'run' | 'collect', contextId: string, files: string[]): Promise<void> {
     const defer = createDefer<void>()
     this.contexts.set(contextId, {
       files,
+      method,
       resolve: () => {
         defer.resolve()
         this.contexts.delete(contextId)

packages/runner/src/index.ts

@@ -1,4 +1,4 @@
-export { startTests, updateTask } from './run'
+export { startTests, updateTask, collectTests } from './run'
 export {
   test,
   it,

packages/runner/src/run.ts

@@ -512,3 +512,14 @@ export async function startTests(paths: string[], runner: VitestRunner) {
 
   return files
 }
+
+async function publicCollect(paths: string[], runner: VitestRunner) {
+  await runner.onBeforeCollect?.(paths)
+
+  const files = await collectTests(paths, runner)
+
+  await runner.onCollected?.(files)
+  return files
+}
+
+export { publicCollect as collectTests }

packages/vitest/LICENSE.md

@@ -438,27 +438,6 @@ License: MIT
 By: Mathias Bynens
 Repository: https://github.com/mathiasbynens/emoji-regex.git
 
-> Copyright Mathias Bynens <https://mathiasbynens.be/>
->
-> Permission is hereby granted, free of charge, to any person obtaining
-> a copy of this software and associated documentation files (the
-> "Software"), to deal in the Software without restriction, including
-> without limitation the rights to use, copy, modify, merge, publish,
-> distribute, sublicense, and/or sell copies of the Software, and to
-> permit persons to whom the Software is furnished to do so, subject to
-> the following conditions:
->
-> The above copyright notice and this permission notice shall be
-> included in all copies or substantial portions of the Software.
->
-> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-> NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-> LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-> OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-> WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
 ---------------------------------------
 
 ## expect-type

packages/vitest/src/browser.ts

@@ -1,4 +1,4 @@
-export { startTests, processError } from '@vitest/runner'
+export { startTests, collectTests, processError } from '@vitest/runner'
 export {
   setupCommonEnv,
   loadDiffConfig,

packages/vitest/src/node/cli/cac.ts

@@ -3,10 +3,10 @@ import cac, { type CAC, type Command } from 'cac'
 import c from 'picocolors'
 import { version } from '../../../package.json' with { type: 'json' }
 import { toArray } from '../../utils/base'
-import type { Vitest, VitestRunMode } from '../../types'
+import type { VitestRunMode } from '../../types'
 import type { CliOptions } from './cli-api'
 import type { CLIOption, CLIOptions as CLIOptionsConfig } from './cli-config'
-import { benchCliOptionsConfig, cliOptionsConfig } from './cli-config'
+import { benchCliOptionsConfig, cliOptionsConfig, collectCliOptionsConfig } from './cli-config'
 
 function addCommand(cli: CAC | Command, name: string, option: CLIOption<any>) {
   const commandName = option.alias || name
@@ -182,6 +182,13 @@ export function createCLI(options: CLIOptions = {}) {
     .command('init <project>', undefined, options)
     .action(init)
 
+  addCliOptions(
+    cli
+      .command('list [...filters]', undefined, options)
+      .action((filters, options) => collect('test', filters, options)),
+    collectCliOptionsConfig,
+  )
+
   cli
     .command('[...filters]', undefined, options)
     .action((filters, options) => start('test', filters, options))
@@ -249,7 +256,7 @@ function normalizeCliOptions(argv: CliOptions): CliOptions {
   return argv
 }
 
-async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<Vitest | undefined> {
+async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<void> {
   try {
     process.title = 'node (vitest)'
   }
@@ -261,7 +268,6 @@ async function start(mode: VitestRunMode, cliFilters: string[], options: CliOpti
     if (!ctx?.shouldKeepServer()) {
       await ctx?.exit()
     }
-    return ctx
   }
   catch (e) {
     const { divider } = await import('../reporters/renderers/utils')
@@ -286,3 +292,44 @@ async function init(project: string) {
   const { create } = await import('../../create/browser/creator')
   await create()
 }
+
+async function collect(mode: VitestRunMode, cliFilters: string[], options: CliOptions): Promise<void> {
+  try {
+    process.title = 'node (vitest)'
+  }
+  catch {}
+
+  try {
+    const { prepareVitest, processCollected } = await import('./cli-api')
+    const ctx = await prepareVitest(mode, {
+      ...normalizeCliOptions(options),
+      watch: false,
+      run: true,
+    })
+
+    const { tests, errors } = await ctx.collect(cliFilters.map(normalize))
+
+    if (errors.length) {
+      console.error('\nThere were unhandled errors during test collection')
+      errors.forEach(e => console.error(e))
+      console.error('\n\n')
+      await ctx.close()
+      return
+    }
+
+    processCollected(ctx, tests, options)
+    await ctx.close()
+  }
+  catch (e) {
+    const { divider } = await import('../reporters/renderers/utils')
+    console.error(`\n${c.red(divider(c.bold(c.inverse(' Collect Error '))))}`)
+    console.error(e)
+    console.error('\n\n')
+
+    if (process.exitCode == null) {
+      process.exitCode = 1
+    }
+
+    process.exit()
+  }
+}