Skip to content

Commit f65fed8

Browse files
ktrzktrz
authored
feat: Comment on PR and auto update comment (#223)
* create PR comment when PR is available * auto-update existing PR comment * add comment-on-alert and gh token to ci jobs
1 parent a6c4f1c commit f65fed8

9 files changed

+178
-23
lines changed

‎.github/workflows/ci-results-repo.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ jobs:
378378
cp ./dist/other-repo/dev/bench/data.js before_data.js
379379
380380
- name: Store benchmark result
381-
uses: benchmark-action/github-action-benchmark@v1
381+
uses: ./
382382
with:
383383
name: Criterion.rs Benchmark
384384
tool: 'cargo'

‎.github/workflows/ci.yml

+21-1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ jobs:
3939
skip-fetch-gh-pages: true
4040
fail-on-alert: true
4141
summary-always: true
42+
github-token: ${{ secrets.GITHUB_TOKEN }}
43+
comment-on-alert: true
4244
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark'
4345

4446
benchmarkjs:
@@ -69,6 +71,8 @@ jobs:
6971
skip-fetch-gh-pages: true
7072
fail-on-alert: true
7173
summary-always: true
74+
github-token: ${{ secrets.GITHUB_TOKEN }}
75+
comment-on-alert: true
7276
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark'
7377

7478
catch2-framework:
@@ -104,6 +108,8 @@ jobs:
104108
skip-fetch-gh-pages: true
105109
fail-on-alert: true
106110
summary-always: true
111+
github-token: ${{ secrets.GITHUB_TOKEN }}
112+
comment-on-alert: true
107113
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
108114

109115
cpp-framework:
@@ -141,6 +147,8 @@ jobs:
141147
skip-fetch-gh-pages: true
142148
fail-on-alert: true
143149
summary-always: true
150+
github-token: ${{ secrets.GITHUB_TOKEN }}
151+
comment-on-alert: true
144152
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark'
145153

146154
go:
@@ -174,6 +182,8 @@ jobs:
174182
skip-fetch-gh-pages: true
175183
fail-on-alert: true
176184
summary-always: true
185+
github-token: ${{ secrets.GITHUB_TOKEN }}
186+
comment-on-alert: true
177187
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark'
178188

179189
java-jmh:
@@ -211,6 +221,8 @@ jobs:
211221
skip-fetch-gh-pages: true
212222
fail-on-alert: true
213223
summary-always: true
224+
github-token: ${{ secrets.GITHUB_TOKEN }}
225+
comment-on-alert: true
214226
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark'
215227

216228
julia-benchmark:
@@ -250,6 +262,8 @@ jobs:
250262
skip-fetch-gh-pages: true
251263
fail-on-alert: true
252264
summary-always: true
265+
github-token: ${{ secrets.GITHUB_TOKEN }}
266+
comment-on-alert: true
253267
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark'
254268

255269
pytest-benchmark:
@@ -286,6 +300,8 @@ jobs:
286300
skip-fetch-gh-pages: true
287301
fail-on-alert: true
288302
summary-always: true
303+
github-token: ${{ secrets.GITHUB_TOKEN }}
304+
comment-on-alert: true
289305
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark'
290306

291307
rust:
@@ -316,6 +332,8 @@ jobs:
316332
output-file-path: examples/rust/output.txt
317333
fail-on-alert: true
318334
summary-always: true
335+
github-token: ${{ secrets.GITHUB_TOKEN }}
336+
comment-on-alert: true
319337
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark'
320338

321339
rust-criterion-rs-framework:
@@ -339,13 +357,15 @@ jobs:
339357
cp ./dev/bench/data.js before_data.js
340358
git checkout -
341359
- name: Store benchmark result
342-
uses: benchmark-action/github-action-benchmark@v1
360+
uses: ./
343361
with:
344362
name: Criterion.rs Benchmark
345363
tool: 'cargo'
346364
output-file-path: examples/criterion-rs/output.txt
347365
fail-on-alert: true
348366
summary-always: true
367+
github-token: ${{ secrets.GITHUB_TOKEN }}
368+
comment-on-alert: true
349369
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark'
350370

351371
only-alert-with-cache:

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
## Unreleased
22
- **fix** Rust benchmarks not comparing to baseline (#235)
3+
- **feat** Comment on PR and auto update comment (#223)
34

45
<a name="v1.19.3"></a>
56
# [v1.19.3](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.19.3) - 02 Feb 2024

‎src/comment/benchmarkCommentTags.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
const BENCHMARK_COMMENT_TAG = 'github-benchmark-action-comment';
2+
3+
export function benchmarkStartTag(commentId: string) {
4+
return `<!-- ${BENCHMARK_COMMENT_TAG}(start): ${commentId} -->`;
5+
}
6+
7+
export function benchmarkEndTag(commentId: string) {
8+
return `<!-- ${BENCHMARK_COMMENT_TAG}(end): ${commentId} -->`;
9+
}
10+
11+
export function wrapBodyWithBenchmarkTags(commentId: string, body: string) {
12+
return `${benchmarkStartTag(commentId)}\n${body}\n${benchmarkEndTag(commentId)}`;
13+
}

‎src/comment/findExistingPRReviewId.ts

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import * as github from '@actions/github';
2+
import { benchmarkStartTag } from './benchmarkCommentTags';
3+
import * as core from '@actions/core';
4+
5+
export async function findExistingPRReviewId(
6+
repoOwner: string,
7+
repoName: string,
8+
pullRequestNumber: number,
9+
benchName: string,
10+
token: string,
11+
) {
12+
core.debug('findExistingPRReviewId start');
13+
const client = github.getOctokit(token);
14+
15+
const existingReviewsResponse = await client.rest.pulls.listReviews({
16+
owner: repoOwner,
17+
repo: repoName,
18+
// eslint-disable-next-line @typescript-eslint/naming-convention
19+
pull_number: pullRequestNumber,
20+
});
21+
22+
const existingReview = existingReviewsResponse.data.find((review) =>
23+
review.body.startsWith(benchmarkStartTag(benchName)),
24+
);
25+
26+
core.debug('findExistingPRReviewId start');
27+
return existingReview?.id;
28+
}

‎src/comment/leaveCommitComment.ts

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import * as github from '@actions/github';
2+
import * as core from '@actions/core';
3+
import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags';
4+
5+
export async function leaveCommitComment(
6+
repoOwner: string,
7+
repoName: string,
8+
commitId: string,
9+
body: string,
10+
commentId: string,
11+
token: string,
12+
) {
13+
core.debug('leaveCommitComment start');
14+
const client = github.getOctokit(token);
15+
const response = await client.rest.repos.createCommitComment({
16+
owner: repoOwner,
17+
repo: repoName,
18+
// eslint-disable-next-line @typescript-eslint/naming-convention
19+
commit_sha: commitId,
20+
body: wrapBodyWithBenchmarkTags(commentId, body),
21+
});
22+
console.log(`Comment was sent to ${response.url}. Response:`, response.status, response.data);
23+
core.debug('leaveCommitComment end');
24+
return response;
25+
}

‎src/comment/leavePRComment.ts

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import * as github from '@actions/github';
2+
import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags';
3+
import { findExistingPRReviewId } from './findExistingPRReviewId';
4+
import * as core from '@actions/core';
5+
6+
export async function leavePRComment(
7+
repoOwner: string,
8+
repoName: string,
9+
pullRequestNumber: number,
10+
body: string,
11+
commentId: string,
12+
token: string,
13+
) {
14+
try {
15+
core.debug('leavePRComment start');
16+
const client = github.getOctokit(token);
17+
18+
const bodyWithTags = wrapBodyWithBenchmarkTags(commentId, body);
19+
20+
const existingCommentId = await findExistingPRReviewId(
21+
repoOwner,
22+
repoName,
23+
pullRequestNumber,
24+
commentId,
25+
token,
26+
);
27+
28+
if (!existingCommentId) {
29+
core.debug('creating new pr comment');
30+
const createReviewResponse = await client.rest.pulls.createReview({
31+
owner: repoOwner,
32+
repo: repoName,
33+
// eslint-disable-next-line @typescript-eslint/naming-convention
34+
pull_number: pullRequestNumber,
35+
event: 'COMMENT',
36+
body: bodyWithTags,
37+
});
38+
39+
console.log(
40+
`Comment was created via ${createReviewResponse.url}. Response:`,
41+
createReviewResponse.status,
42+
createReviewResponse.data,
43+
);
44+
45+
core.debug('leavePRComment end');
46+
return createReviewResponse;
47+
}
48+
49+
core.debug('updating existing pr comment');
50+
const updateReviewResponse = await client.rest.pulls.updateReview({
51+
owner: repoOwner,
52+
repo: repoName,
53+
// eslint-disable-next-line @typescript-eslint/naming-convention
54+
pull_number: pullRequestNumber,
55+
// eslint-disable-next-line @typescript-eslint/naming-convention
56+
review_id: existingCommentId,
57+
body: bodyWithTags,
58+
});
59+
60+
console.log(
61+
`Comment was updated via ${updateReviewResponse.url}. Response:`,
62+
updateReviewResponse.status,
63+
updateReviewResponse.data,
64+
);
65+
core.debug('leavePRComment end');
66+
67+
return updateReviewResponse;
68+
} catch (err) {
69+
console.log('error', err);
70+
throw err;
71+
}
72+
}

‎src/write.ts

+12-20
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import * as git from './git';
77
import { Benchmark, BenchmarkResult } from './extract';
88
import { Config, ToolType } from './config';
99
import { DEFAULT_INDEX_HTML } from './default_index_html';
10+
import { leavePRComment } from './comment/leavePRComment';
11+
import { leaveCommitComment } from './comment/leaveCommitComment';
1012

1113
export type BenchmarkSuites = { [name: string]: Benchmark[] };
1214
export interface DataJson {
@@ -236,24 +238,15 @@ function buildAlertComment(
236238
return lines.join('\n');
237239
}
238240

239-
async function leaveComment(commitId: string, body: string, token: string) {
241+
async function leaveComment(commitId: string, body: string, commentId: string, token: string) {
240242
core.debug('Sending comment:\n' + body);
241243

242244
const repoMetadata = getCurrentRepoMetadata();
243-
const repoUrl = repoMetadata.html_url ?? '';
244-
const client = github.getOctokit(token);
245-
const res = await client.rest.repos.createCommitComment({
246-
owner: repoMetadata.owner.login,
247-
repo: repoMetadata.name,
248-
// eslint-disable-next-line @typescript-eslint/naming-convention
249-
commit_sha: commitId,
250-
body,
251-
});
252-
253-
const commitUrl = `${repoUrl}/commit/${commitId}`;
254-
console.log(`Comment was sent to ${commitUrl}. Response:`, res.status, res.data);
245+
const pr = github.context.payload.pull_request;
255246

256-
return res;
247+
return await (pr?.number
248+
? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token)
249+
: leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token));
257250
}
258251

259252
async function handleComment(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) {
@@ -272,7 +265,7 @@ async function handleComment(benchName: string, curSuite: Benchmark, prevSuite:
272265

273266
const body = buildComment(benchName, curSuite, prevSuite);
274267

275-
await leaveComment(curSuite.commit.id, body, githubToken);
268+
await leaveComment(curSuite.commit.id, body, `${benchName} Summary`, githubToken);
276269
}
277270

278271
async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) {
@@ -292,14 +285,13 @@ async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Be
292285
core.debug(`Found ${alerts.length} alerts`);
293286
const body = buildAlertComment(alerts, benchName, curSuite, prevSuite, alertThreshold, alertCommentCcUsers);
294287
let message = body;
295-
let url = null;
296288

297289
if (commentOnAlert) {
298290
if (!githubToken) {
299291
throw new Error("'comment-on-alert' input is set but 'github-token' input is not set");
300292
}
301-
const res = await leaveComment(curSuite.commit.id, body, githubToken);
302-
url = res.data.html_url;
293+
const res = await leaveComment(curSuite.commit.id, body, `${benchName} Alert`, githubToken);
294+
const url = res.data.html_url;
303295
message = body + `\nComment was generated at ${url}`;
304296
}
305297

@@ -364,7 +356,7 @@ function addBenchmarkToDataJson(
364356
return prevBench;
365357
}
366358

367-
function isRemoteRejectedError(err: unknown) {
359+
function isRemoteRejectedError(err: unknown): err is Error {
368360
if (err instanceof Error) {
369361
return ['[remote rejected]', '[rejected]'].some((l) => err.message.includes(l));
370362
}
@@ -443,7 +435,7 @@ async function writeBenchmarkToGitHubPagesWithRetry(
443435
console.log(
444436
`Automatically pushed the generated commit to ${ghPagesBranch} branch since 'auto-push' is set to true`,
445437
);
446-
} catch (err: any) {
438+
} catch (err: unknown) {
447439
if (!isRemoteRejectedError(err)) {
448440
throw err;
449441
}

‎test/write.spec.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { Benchmark } from '../src/extract';
88
import { DataJson, writeBenchmark } from '../src/write';
99
import { expect } from '@jest/globals';
1010
import { FakedOctokit, fakedRepos } from './fakedOctokit';
11+
import { wrapBodyWithBenchmarkTags } from '../src/comment/benchmarkCommentTags';
1112

1213
const ok: (x: any, msg?: string) => asserts x = (x, msg) => {
1314
try {
@@ -771,7 +772,10 @@ describe.each(['https://github.com', 'https://github.enterprise.corp'])('writeBe
771772
// Last line is appended only for failure message
772773
const messageLines = caughtError.message.split('\n');
773774
ok(messageLines.length > 0);
774-
const expectedMessage = messageLines.slice(0, -1).join('\n');
775+
const expectedMessage = wrapBodyWithBenchmarkTags(
776+
'Test benchmark Alert',
777+
messageLines.slice(0, -1).join('\n'),
778+
);
775779
ok(
776780
fakedRepos.spyOpts.length > 0,
777781
`len: ${fakedRepos.spyOpts.length}, caught: ${caughtError.message}`,

0 commit comments

Comments
 (0)