Throw error when mocha parallel is set to true (#833)

Author: cgewecke

README.md

@@ -122,6 +122,7 @@ Common problems & questions:
 + [Running out of time][6]
 + [Running out of stack][1002] (Stack too deep)
 + [Running out of memory][5]
++ [Running in parallel (in CI)][1003]
 
 ## Example reports
 + [metacoin][9] (Istanbul HTML)
@@ -216,4 +217,5 @@ $ yarn
 [39]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#generating-a-test-matrix
 [1001]: https://docs.soliditylang.org/en/v0.8.0/using-the-compiler.html#input-description
 [1002]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/faq.md#running-out-of-stack
+[1003]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/advanced.md#parallelization-in-ci
 

docs/advanced.md

@@ -125,8 +125,24 @@ In combination these data sets can be passed to Joran Honig's [tarantula][29] to
 a fault localization algorithm to generate 'suspiciousness' ratings for each line of
 Solidity code in your project.
 
+## Parallelization in CI
+
+Coverage does not work with the Hardhat's mocha parallel mode. However, it *is* possible to parallelize coverage in CI environments that support complex workflows. The core idea is to
+
++ partition the set of test files passed to the coverage task
++ split coverage into several concurrent jobs, passing test file targets as arguments using the `--testfiles` command line flag
++ cache the coverage results in shared storage as each job completes
++ combine results in a final step (using the [instanbul-combine-updated][30] tool)
+
+There's a nice example of this being done in CircleCI [at Synthetix, here][31].
+
+:bulb: **Pro Tip**: Codecov CI will automatically combine coverage reports sent to them as a batch - if you're using that service you don't need to do this yourself.
+
+
 [22]: https://github.com/JoranHonig/vertigo#vertigo
 [23]: http://spideruci.org/papers/jones05.pdf
 [25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
 [27]: https://mochajs.org/api/reporters_json.js.html
 [29]: https://github.com/JoranHonig/tarantula
+[30]: https://www.npmjs.com/package/istanbul-combine-updated
+[31]: https://github.com/Synthetixio/synthetix/blob/bd54f4e9cfd1529d8ea2e5a7b4d0be4c988e1a03/.circleci/src/jobs/job-unit-tests-coverage.yml

plugins/resources/plugin.ui.js

@@ -56,6 +56,11 @@ class PluginUI extends UI {
                          `${c.red('the path you specified for it is wrong.')}`,
 
       'tests-fail': `${x} ${c.bold(args[0])} ${c.red('test(s) failed under coverage.')}`,
+
+      'mocha-parallel-fail': `${c.red('Coverage cannot be run in mocha parallel mode. ')}` +
+                             `${c.red('Set \`mocha: { parallel: false }\` in .solcover.js ')}` +
+                             `${c.red('to disable the option for the coverage task. ')}` +
+                             `${c.red('See the solidity-coverage README FAQ for info on parallelizing coverage in CI.')}`,
     }
 
 

plugins/resources/plugin.utils.js

@@ -232,6 +232,12 @@ function loadSolcoverJS(config={}){
     );
   }
 
+  // Per fvictorio recommendation in #691
+  if (config.mocha.parallel) {
+    const message = ui.generate('mocha-parallel-fail');
+    throw new Error(message);
+  }
+
   return coverageConfig;
 }
 

test/integration/projects/parallel/.solcover.js

@@ -0,0 +1,7 @@
+module.exports = {
+  "silent": false,
+  "istanbulReporter": [ "json-summary", "text"],
+  "mocha": {
+    parallel: true // manually tested that setting to false overrides HH config
+  },
+}

test/integration/projects/parallel/contracts/ContractA.sol

@@ -0,0 +1,10 @@
+pragma solidity ^0.7.0;
+
+
+contract ContractA {
+  uint x;
+
+  function set() public {
+    x = 5;
+  }
+}

test/integration/projects/parallel/hardhat.config.js

@@ -0,0 +1,8 @@
+require(__dirname + "/../plugins/nomiclabs.plugin");
+
+module.exports = {
+  mocha: {
+    parallel: true
+  },
+  logger: process.env.SILENT ? { log: () => {} } : console,
+};

test/units/hardhat/errors.js

@@ -202,4 +202,21 @@ describe('Hardhat Plugin: error cases', function() {
 
     verify.coverageNotGenerated(hardhatConfig);
   });
+
+  it('mocha parallel option is true', async function(){
+    mock.installFullProject('parallel');
+    mock.hardhatSetupEnv(this);
+
+    try {
+      await this.env.run("coverage");
+      assert.fail()
+    } catch(err){
+      assert(
+        err.message.includes('Coverage cannot be run in mocha parallel mode'),
+        `Should notify when mocha parallel flag is set:: ${err.message}`
+      );
+    }
+
+    verify.coverageNotGenerated(hardhatConfig);
+  })
 })