|
| 1 | +import * as fs from 'fs/promises'; |
| 2 | +import path from 'path'; |
| 3 | +import type { |
| 4 | + TestCase, |
| 5 | + TestFile, |
| 6 | + TestRun, |
| 7 | + TestSuite, |
| 8 | +} from './shared/test-reports'; |
| 9 | +import { |
| 10 | + consoleBold, |
| 11 | + formatTime, |
| 12 | + normalizeTestPath, |
| 13 | + XML, |
| 14 | +} from './shared/utils'; |
| 15 | + |
| 16 | +async function main() { |
| 17 | + const env = { |
| 18 | + OWNER: process.env.OWNER || 'metamask', |
| 19 | + REPOSITORY: process.env.REPOSITORY || 'metamask-extension', |
| 20 | + BRANCH: process.env.BRANCH || 'main', |
| 21 | + TEST_SUMMARY_PATH: |
| 22 | + process.env.TEST_SUMMARY_PATH || 'test/test-results/summary.md', |
| 23 | + TEST_RESULTS_PATH: process.env.TEST_RESULTS_PATH || 'test/test-results/e2e', |
| 24 | + TEST_RUNS_PATH: |
| 25 | + process.env.TEST_RUNS_PATH || 'test/test-results/test-runs.json', |
| 26 | + RUN_ID: process.env.RUN_ID ? +process.env.RUN_ID : 0, |
| 27 | + PR_NUMBER: process.env.PR_NUMBER ? +process.env.PR_NUMBER : 0, |
| 28 | + GITHUB_ACTIONS: process.env.GITHUB_ACTIONS === 'true', |
| 29 | + }; |
| 30 | + |
| 31 | + let summary = ''; |
| 32 | + const core = env.GITHUB_ACTIONS |
| 33 | + ? await import('@actions/core') |
| 34 | + : { |
| 35 | + summary: { |
| 36 | + addRaw: (text: string) => { |
| 37 | + summary += text; |
| 38 | + }, |
| 39 | + write: async () => await fs.writeFile(env.TEST_SUMMARY_PATH, summary), |
| 40 | + }, |
| 41 | + setFailed: (msg: string) => console.error(msg), |
| 42 | + }; |
| 43 | + |
| 44 | + try { |
| 45 | + const testRuns: TestRun[] = []; |
| 46 | + const filenames = await fs.readdir(env.TEST_RESULTS_PATH); |
| 47 | + |
| 48 | + for (const filename of filenames) { |
| 49 | + const file = await fs.readFile( |
| 50 | + path.join(env.TEST_RESULTS_PATH, filename), |
| 51 | + 'utf8', |
| 52 | + ); |
| 53 | + const results = await XML.parse(file); |
| 54 | + |
| 55 | + for (const suite of results.testsuites.testsuite || []) { |
| 56 | + if (!suite.testcase || !suite.$.file) continue; |
| 57 | + const tests = +suite.$.tests; |
| 58 | + const failed = +suite.$.failures; |
| 59 | + const skipped = tests - suite.testcase.length; |
| 60 | + const passed = tests - failed - skipped; |
| 61 | + |
| 62 | + const testSuite: TestSuite = { |
| 63 | + name: suite.$.name, |
| 64 | + job: { |
| 65 | + name: suite.properties?.[0].property?.[0]?.$.value ?? '', |
| 66 | + id: suite.properties?.[0].property?.[1]?.$.value ?? '', |
| 67 | + }, |
| 68 | + date: new Date(suite.$.timestamp), |
| 69 | + tests, |
| 70 | + passed, |
| 71 | + failed, |
| 72 | + skipped, |
| 73 | + time: +suite.$.time * 1000, // convert to ms, |
| 74 | + attempts: [], |
| 75 | + testCases: [], |
| 76 | + }; |
| 77 | + |
| 78 | + for (const test of suite.testcase || []) { |
| 79 | + const testCase: TestCase = { |
| 80 | + name: test.$.name, |
| 81 | + time: +test.$.time * 1000, // convert to ms |
| 82 | + status: test.failure ? 'failed' : 'passed', |
| 83 | + error: test.failure ? test.failure[0]._ : undefined, |
| 84 | + }; |
| 85 | + testSuite.testCases.push(testCase); |
| 86 | + } |
| 87 | + |
| 88 | + const testFile: TestFile = { |
| 89 | + path: normalizeTestPath(suite.$.file), |
| 90 | + tests: testSuite.tests, |
| 91 | + passed: testSuite.passed, |
| 92 | + failed: testSuite.failed, |
| 93 | + skipped: testSuite.skipped, |
| 94 | + time: testSuite.time, |
| 95 | + testSuites: [testSuite], |
| 96 | + }; |
| 97 | + |
| 98 | + const testRun: TestRun = { |
| 99 | + // regex to remove the shard number from the job name |
| 100 | + name: testSuite.job.name.replace(/\s+\(\d+\)$/, ''), |
| 101 | + testFiles: [testFile], |
| 102 | + }; |
| 103 | + |
| 104 | + const existingRun = testRuns.find((run) => run.name === testRun.name); |
| 105 | + if (existingRun) { |
| 106 | + const existingFile = existingRun.testFiles.find( |
| 107 | + (file) => file.path === testFile.path, |
| 108 | + ); |
| 109 | + if (existingFile) { |
| 110 | + existingFile.testSuites.push(testSuite); |
| 111 | + } else { |
| 112 | + existingRun.testFiles.push(testFile); |
| 113 | + } |
| 114 | + } else { |
| 115 | + testRuns.push(testRun); |
| 116 | + } |
| 117 | + } |
| 118 | + } |
| 119 | + |
| 120 | + for (const testRun of testRuns) { |
| 121 | + for (const testFile of testRun.testFiles) { |
| 122 | + // Group test suites by name |
| 123 | + const suitesByName: Record<string, TestSuite[]> = {}; |
| 124 | + |
| 125 | + for (const suite of testFile.testSuites) { |
| 126 | + if (!suitesByName[suite.name]) { |
| 127 | + suitesByName[suite.name] = []; |
| 128 | + } |
| 129 | + suitesByName[suite.name].push(suite); |
| 130 | + } |
| 131 | + |
| 132 | + // Determine the latest test suite by date and nest attempts |
| 133 | + const attempts: TestSuite[][] = []; |
| 134 | + for (const suites of Object.values(suitesByName)) { |
| 135 | + suites.sort((a, b) => b.date.getTime() - a.date.getTime()); // sort newest first |
| 136 | + const [latest, ...otherAttempts] = suites; |
| 137 | + latest.attempts = otherAttempts; |
| 138 | + attempts.push(otherAttempts); |
| 139 | + } |
| 140 | + |
| 141 | + // Remove the nested attempts from the top-level list |
| 142 | + const attemptSet = new Set(attempts.flat()); |
| 143 | + testFile.testSuites = testFile.testSuites.filter( |
| 144 | + (suite) => !attemptSet.has(suite), |
| 145 | + ); |
| 146 | + |
| 147 | + const total = testFile.testSuites.reduce( |
| 148 | + (acc, suite) => ({ |
| 149 | + tests: acc.tests + suite.tests, |
| 150 | + passed: acc.passed + suite.passed, |
| 151 | + failed: acc.failed + suite.failed, |
| 152 | + skipped: acc.skipped + suite.skipped, |
| 153 | + time: acc.time + suite.time, |
| 154 | + }), |
| 155 | + { tests: 0, passed: 0, failed: 0, skipped: 0, time: 0 }, |
| 156 | + ); |
| 157 | + |
| 158 | + testFile.tests = total.tests; |
| 159 | + testFile.passed = total.passed; |
| 160 | + testFile.failed = total.failed; |
| 161 | + testFile.skipped = total.skipped; |
| 162 | + testFile.time = total.time; |
| 163 | + } |
| 164 | + |
| 165 | + testRun.testFiles.sort((a, b) => a.path.localeCompare(b.path)); |
| 166 | + |
| 167 | + const title = `<strong>${testRun.name}</strong>`; |
| 168 | + |
| 169 | + if (testRun.testFiles.length) { |
| 170 | + const total = testRun.testFiles.reduce( |
| 171 | + (acc, file) => ({ |
| 172 | + tests: acc.tests + file.tests, |
| 173 | + passed: acc.passed + file.passed, |
| 174 | + failed: acc.failed + file.failed, |
| 175 | + skipped: acc.skipped + file.skipped, |
| 176 | + time: acc.time + file.time, |
| 177 | + }), |
| 178 | + { tests: 0, passed: 0, failed: 0, skipped: 0, time: 0 }, |
| 179 | + ); |
| 180 | + |
| 181 | + if (total.failed > 0) { |
| 182 | + if (testRun.name) console.log(`${consoleBold(title)} ❌`); |
| 183 | + core.summary.addRaw(`\n<details open>\n`); |
| 184 | + core.summary.addRaw(`\n<summary>${title} ❌</summary>\n`); |
| 185 | + } else { |
| 186 | + if (testRun.name) console.log(`${consoleBold(title)} ✅`); |
| 187 | + core.summary.addRaw(`\n<details>\n`); |
| 188 | + core.summary.addRaw(`\n<summary>${title} ✅</summary>\n`); |
| 189 | + } |
| 190 | + |
| 191 | + const times = testRun.testFiles |
| 192 | + .map((file) => |
| 193 | + file.testSuites.map((suite) => ({ |
| 194 | + start: suite.date.getTime(), |
| 195 | + end: suite.date.getTime() + suite.time, |
| 196 | + })), |
| 197 | + ) |
| 198 | + .flat(); |
| 199 | + const earliestStart = Math.min(...times.map((t) => t.start)); |
| 200 | + const latestEnd = Math.max(...times.map((t) => t.end)); |
| 201 | + const executionTime = latestEnd - earliestStart; |
| 202 | + |
| 203 | + const conclusion = `<strong>${total.tests}</strong> ${ |
| 204 | + total.tests === 1 ? 'test was' : 'tests were' |
| 205 | + } completed in <strong>${formatTime( |
| 206 | + executionTime, |
| 207 | + )}</strong> with <strong>${total.passed}</strong> passed, <strong>${ |
| 208 | + total.failed |
| 209 | + }</strong> failed and <strong>${total.skipped}</strong> skipped.`; |
| 210 | + |
| 211 | + console.log(consoleBold(conclusion)); |
| 212 | + core.summary.addRaw(`\n${conclusion}\n`); |
| 213 | + |
| 214 | + if (total.failed) { |
| 215 | + console.error(`\n❌ Failed tests\n`); |
| 216 | + core.summary.addRaw(`\n#### ❌ Failed tests\n`); |
| 217 | + core.summary.addRaw( |
| 218 | + `\n<hr style="height: 1px; margin-top: -5px; margin-bottom: 10px;">\n`, |
| 219 | + ); |
| 220 | + for (const file of testRun.testFiles) { |
| 221 | + if (file.failed === 0) continue; |
| 222 | + console.error(file.path); |
| 223 | + core.summary.addRaw( |
| 224 | + `\n#### [${file.path}](https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${file.path})\n`, |
| 225 | + ); |
| 226 | + for (const suite of file.testSuites) { |
| 227 | + if (suite.failed === 0) continue; |
| 228 | + if (suite.job.name && suite.job.id && env.RUN_ID) { |
| 229 | + core.summary.addRaw( |
| 230 | + `\n##### Job: [${suite.job.name}](https://github.com/${ |
| 231 | + env.OWNER |
| 232 | + }/${env.REPOSITORY}/actions/runs/${env.RUN_ID}/job/${ |
| 233 | + suite.job.id |
| 234 | + }${env.PR_NUMBER ? `?pr=${env.PR_NUMBER}` : ''})\n`, |
| 235 | + ); |
| 236 | + } |
| 237 | + for (const test of suite.testCases) { |
| 238 | + if (test.status !== 'failed') continue; |
| 239 | + console.error(` ${test.name}`); |
| 240 | + console.error(` ${test.error}\n`); |
| 241 | + core.summary.addRaw(`\n##### ${test.name}\n`); |
| 242 | + core.summary.addRaw(`\n\`\`\`js\n${test.error}\n\`\`\`\n`); |
| 243 | + } |
| 244 | + } |
| 245 | + } |
| 246 | + } |
| 247 | + |
| 248 | + const rows = testRun.testFiles.map((file) => ({ |
| 249 | + 'Test file': file.path, |
| 250 | + Passed: file.passed ? `${file.passed} ✅` : '', |
| 251 | + Failed: file.failed ? `${file.failed} ❌` : '', |
| 252 | + Skipped: file.skipped ? `${file.skipped} ⏩` : '', |
| 253 | + Time: formatTime(file.time), |
| 254 | + })); |
| 255 | + |
| 256 | + const columns = Object.keys(rows[0]); |
| 257 | + const header = `| ${columns.join(' | ')} |`; |
| 258 | + const alignment = '| :--- | ---: | ---: | ---: | ---: |'; |
| 259 | + const body = rows |
| 260 | + .map((row) => { |
| 261 | + const data = { |
| 262 | + ...row, |
| 263 | + 'Test file': `[${row['Test file']}](https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${row['Test file']})`, |
| 264 | + }; |
| 265 | + return `| ${Object.values(data).join(' | ')} |`; |
| 266 | + }) |
| 267 | + .join('\n'); |
| 268 | + const table = [header, alignment, body].join('\n'); |
| 269 | + |
| 270 | + console.table(rows); |
| 271 | + core.summary.addRaw(`\n${table}\n`); |
| 272 | + } else { |
| 273 | + core.summary.addRaw(`\n<details open>\n`); |
| 274 | + core.summary.addRaw(`<summary>${title}</summary>\n`); |
| 275 | + console.log('No tests found'); |
| 276 | + core.summary.addRaw('No tests found'); |
| 277 | + } |
| 278 | + console.log(); |
| 279 | + core.summary.addRaw(`</details>\n`); |
| 280 | + } |
| 281 | + |
| 282 | + await core.summary.write(); |
| 283 | + await fs.writeFile(env.TEST_RUNS_PATH, JSON.stringify(testRuns, null, 2)); |
| 284 | + } catch (error) { |
| 285 | + core.setFailed(`Error creating the test report: ${error}`); |
| 286 | + } |
| 287 | +} |
| 288 | + |
| 289 | +main(); |
0 commit comments