Add --coverage-lcov and --branch-coverage options

pkgs/test/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.27.0
+
+* Add `--coverage-lcov` and `--branch-coverage` options to `dart test`.
+
 ## 1.26.2
 
 * Graduate native assets from experiment to preview.

pkgs/test/README.md

@@ -258,35 +258,27 @@ The available options for the `--reporter` flag are:
 ### Collecting Code Coverage
 
 To collect code coverage, you can run tests with the `--coverage <directory>`
-argument. The directory specified can be an absolute or relative path.
+argument or the `--coverage-lcov <file>` argument. The `--coverage` option
+outputs a json formatted report per test suite, whereas the `--coverage-lcov`
+option merges all the test coverage into a single LCOV file.
+The directory or file specified can be an absolute or relative path.
 If a directory does not exist at the path specified, a directory will be
 created. If a directory does exist, files may be overwritten with the latest
 coverage data, if they conflict.
 
-This option will enable code coverage collection on a suite-by-suite basis,
-and the resulting coverage files will be outputted in the directory specified.
-The files can then be formatted using the `package:coverage`
-`format_coverage` executable.
-
 Coverage gathering is currently only implemented for tests run on the Dart VM or
 Chrome.
 
 Here's an example of how to run tests and format the collected coverage to LCOV:
 
 ```shell
-## Run Dart tests and output them at directory `./coverage`:
-dart run test --coverage=./coverage
-
-## Activate package `coverage` (if needed):
-dart pub global activate coverage
-
-## Format collected coverage to LCOV (only for directory "lib")
-dart pub global run coverage:format_coverage --packages=.dart_tool/package_config.json --report-on=lib --lcov -o ./coverage/lcov.info -i ./coverage
+## Run Dart tests and output coverage info to `./coverage/lcov.info`:
+dart run test --coverage-lcov=./coverage/lcov.info
 
-## Generate LCOV report:
+## Generate a human readable report:
 genhtml -o ./coverage/report ./coverage/lcov.info
 
-## Open the HTML coverage report:
+## Open the coverage report:
 open ./coverage/report/index.html
 ```
 

pkgs/test/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test
-version: 1.26.2
+version: 1.27.0
 description: >-
   A full featured library for writing and running Dart tests across platforms.
 repository: https://github.com/dart-lang/test/tree/master/pkgs/test
@@ -14,7 +14,7 @@ dependencies:
   async: ^2.5.0
   boolean_selector: ^2.1.0
   collection: ^1.15.0
-  coverage: ^1.0.1
+  coverage: ^1.15.0
   http_multi_server: ^3.0.0
   io: ^1.0.0
   js: '>=0.6.4 <0.8.0'
@@ -37,7 +37,7 @@ dependencies:
 
   # Use an exact version until the test_api and test_core package are stable.
   test_api: 0.7.6
-  test_core: 0.6.11
+  test_core: 0.6.12
 
   typed_data: ^1.3.0
   web_socket_channel: '>=2.0.0 <4.0.0'

pkgs/test/test/io.dart

@@ -21,10 +21,6 @@ final Future<String> packageDir =
   return dir;
 });
 
-/// The path to the `pub` executable in the current Dart SDK.
-final _pubPath = p.absolute(p.join(p.dirname(Platform.resolvedExecutable),
-    Platform.isWindows ? 'pub.bat' : 'pub'));
-
 /// The platform-specific message emitted when a nonexistent file is loaded.
 final String noSuchFileMessage = Platform.isWindows
     ? 'The system cannot find the file specified.'
@@ -151,7 +147,8 @@ Future<TestProcess> runDart(Iterable<String> args,
 /// Runs Pub.
 Future<TestProcess> runPub(Iterable<String> args,
     {Map<String, String>? environment}) {
-  return TestProcess.start(_pubPath, args,
+  return TestProcess.start(
+      p.absolute(Platform.resolvedExecutable), ['pub', ...args],
       workingDirectory: d.sandbox,
       environment: environment,
       description: 'pub ${args.first}');

pkgs/test/test/runner/coverage_test.dart

@@ -8,6 +8,7 @@ library;
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:coverage/coverage.dart';
 import 'package:path/path.dart' as p;
 import 'package:test/test.dart';
 import 'package:test_descriptor/test_descriptor.dart' as d;
@@ -20,46 +21,126 @@ void main() {
 
   group('with the --coverage flag,', () {
     late Directory coverageDirectory;
+    late d.DirectoryDescriptor packageDirectory;
 
-    Future<void> validateCoverage(TestProcess test, String coveragePath) async {
+    Future<void> validateTest(TestProcess test) async {
       expect(test.stdout, emitsThrough(contains('+1: All tests passed!')));
       await test.shouldExit(0);
+    }
+
+    Future<Map<String, HitMap>> validateCoverage(
+        TestProcess test, String coveragePath) async {
+      await validateTest(test);
 
       final coverageFile = File(p.join(coverageDirectory.path, coveragePath));
       final coverage = await coverageFile.readAsString();
-      final jsonCoverage = json.decode(coverage);
-      expect(jsonCoverage['coverage'], isNotEmpty);
+      final jsonCoverage = json.decode(coverage)['coverage'] as List<dynamic>;
+      expect(jsonCoverage, isNotEmpty);
+
+      return HitMap.parseJson(jsonCoverage.cast<Map<String, dynamic>>());
     }
 
     setUp(() async {
-      await d.file('test.dart', '''
-        import 'package:test/test.dart';
-
-        void main() {
-          test("test 1", () {
-            expect(true, isTrue);
-          });
-        }
-      ''').create();
-
       coverageDirectory =
           await Directory.systemTemp.createTemp('test_coverage');
+
+      packageDirectory = d.dir(d.sandbox, [
+        d.dir('lib', [
+          d.file('calculate.dart', '''
+            int calculate(int x) {
+              if (x % 2 == 0) {
+                return x * 2;
+              } else {
+                return x * 3;
+              }
+            }
+          '''),
+        ]),
+        d.dir('test', [
+          d.file('test.dart', '''
+            import 'package:fake_package/calculate.dart';
+            import 'package:test/test.dart';
+
+            void main() {
+              test('test 1', () {
+                expect(calculate(6), 12);
+              });
+            }
+          ''')
+        ]),
+        d.file('pubspec.yaml', '''
+name: fake_package
+version: 1.0.0
+environment:
+  sdk: ^3.5.0
+dev_dependencies:
+  test: ^1.26.2
+        '''),
+      ]);
+      await packageDirectory.create();
     });
 
     tearDown(() async {
       await coverageDirectory.delete(recursive: true);
     });
 
     test('gathers coverage for VM tests', () async {
-      var test =
-          await runTest(['--coverage', coverageDirectory.path, 'test.dart']);
-      await validateCoverage(test, 'test.dart.vm.json');
+      await (await runPub(['get'])).shouldExit(0);
+      var test = await runTest(
+          ['--coverage', coverageDirectory.path, 'test/test.dart'],
+          packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      final coverage = await validateCoverage(test, 'test/test.dart.vm.json');
+      final hitmap = coverage['package:fake_package/calculate.dart']!;
+      expect(hitmap.lineHits, {1: 1, 2: 2, 3: 1, 5: 0});
+      expect(hitmap.funcHits, isNull);
+      expect(hitmap.branchHits, isNull);
+    });
+
+    test('gathers branch coverage for VM tests', () async {
+      await (await runPub(['get'])).shouldExit(0);
+      var test = await runTest([
+        '--coverage',
+        coverageDirectory.path,
+        '--branch-coverage',
+        'test/test.dart'
+      ], vmArgs: [
+        '--branch-coverage'
+      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      final coverage = await validateCoverage(test, 'test/test.dart.vm.json');
+      final hitmap = coverage['package:fake_package/calculate.dart']!;
+      expect(hitmap.lineHits, {1: 1, 2: 2, 3: 1, 5: 0});
+      expect(hitmap.funcHits, isNull);
+      expect(hitmap.branchHits, {1: 1, 2: 1, 4: 0});
+    });
+
+    test('gathers lcov coverage for VM tests', () async {
+      await (await runPub(['get'])).shouldExit(0);
+      final lcovFile = p.join(coverageDirectory.path, 'lcov.info');
+      var test = await runTest(['--coverage-lcov', lcovFile, 'test/test.dart'],
+          packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      await validateTest(test);
+      expect(File(lcovFile).readAsStringSync(), '''
+SF:${p.join(d.sandbox, 'lib', 'calculate.dart')}
+DA:1,1
+DA:2,2
+DA:3,1
+DA:5,0
+LF:4
+LH:3
+end_of_record
+''');
     });
 
     test('gathers coverage for Chrome tests', () async {
-      var test = await runTest(
-          ['--coverage', coverageDirectory.path, 'test.dart', '-p', 'chrome']);
-      await validateCoverage(test, 'test.dart.chrome.json');
+      await (await runPub(['get'])).shouldExit(0);
+      var test = await runTest([
+        '--coverage',
+        coverageDirectory.path,
+        'test/test.dart',
+        '-p',
+        'chrome'
+      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      await validateCoverage(test, 'test/test.dart.chrome.json');
     });
 
     test(

pkgs/test/test/runner/runner_test.dart

@@ -92,6 +92,10 @@ $_runtimeCompilers
     --debug                           Run the VM and Chrome tests in debug mode.
     --coverage=<directory>            Gather coverage and output it to the specified directory.
                                       Implies --debug.
+    --coverage-lcov=<file>            Gather coverage and output an lcov report to the specified file.
+                                      Implies --debug.
+    --branch-coverage                 Include branch coverage information in the coverage report.
+                                      Must be paired with --coverage or --coverage-lcov.
     --[no-]chain-stack-traces         Use chained stack traces to provide greater exception details
                                       especially for asynchronous code. It may be useful to disable
                                       to provide improved test performance but at the cost of

pkgs/test/test/utils.dart

@@ -121,6 +121,7 @@ Engine declareEngine(
   void Function() body, {
   bool runSkipped = false,
   String? coverage,
+  String? coverageLcov,
   bool stopOnFirstFailure = false,
 }) {
   var declarer = Declarer()..declare(body);
@@ -133,6 +134,7 @@ Engine declareEngine(
           suitePlatform)
     ],
     coverage: coverage,
+    coverageLcov: coverageLcov,
     stopOnFirstFailure: stopOnFirstFailure,
   );
 }
@@ -199,6 +201,8 @@ Configuration configuration(
         String? reporter,
         Map<String, String>? fileReporters,
         String? coverage,
+        String? coverageLcov,
+        bool? branchCoverage,
         int? concurrency,
         int? shardIndex,
         int? totalShards,
@@ -249,6 +253,8 @@ Configuration configuration(
         reporter: reporter,
         fileReporters: fileReporters,
         coverage: coverage,
+        coverageLcov: coverageLcov,
+        branchCoverage: branchCoverage,
         concurrency: concurrency,
         shardIndex: shardIndex,
         totalShards: totalShards,

pkgs/test_core/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.6.12
+
+* Add `--coverage-lcov` and `--branch-coverage` options to `dart test`.
+
 ## 0.6.11
 
 * Graduate native assets from experiment to preview.

pkgs/test_core/lib/src/runner.dart

@@ -77,6 +77,7 @@ class Runner {
         var engine = Engine(
           concurrency: config.concurrency,
           coverage: config.coverage,
+          coverageLcov: config.coverageLcov,
           testRandomizeOrderingSeed: config.testRandomizeOrderingSeed,
           stopOnFirstFailure: config.stopOnFirstFailure,
         );