Skip to content

Commit 2b82b9a

Browse files
ktrzktrz
authored
fix: Catch2 v.3.5.0 changed output format (#247)
* create Catch2 v3 example * add test case for catch2 v3 * modify extract header regexp to work with v3
1 parent e473152 commit 2b82b9a

17 files changed

+366
-20
lines changed

‎.github/workflows/catch2.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ permissions:
1010

1111
jobs:
1212
benchmark:
13-
name: Run C++ benchmark example
13+
name: Run Catch2 C++ Benchmark Framework example (v3.x)
1414
runs-on: ubuntu-20.04
1515
steps:
1616
- uses: actions/checkout@v4
@@ -25,7 +25,7 @@ jobs:
2525
- name: Store benchmark result
2626
uses: benchmark-action/github-action-benchmark@v1
2727
with:
28-
name: Catch2 Benchmark
28+
name: Catch2 Benchmark (v3)
2929
tool: 'catch2'
3030
output-file-path: examples/catch2/benchmark_result.txt
3131
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -34,12 +34,12 @@ jobs:
3434
alert-threshold: '200%'
3535
comment-on-alert: true
3636
fail-on-alert: true
37-
alert-comment-cc-users: '@bernedom,@ktrz'
37+
alert-comment-cc-users: '@ktrz'
3838

3939
- name: Store benchmark result - separate results repo
4040
uses: benchmark-action/github-action-benchmark@v1
4141
with:
42-
name: Catch2 Benchmark
42+
name: Catch2 Benchmark (v3)
4343
tool: 'catch2'
4444
output-file-path: examples/catch2/benchmark_result.txt
4545
github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
@@ -48,5 +48,5 @@ jobs:
4848
alert-threshold: '200%'
4949
comment-on-alert: true
5050
fail-on-alert: true
51-
alert-comment-cc-users: '@bernedom,@ktrz'
51+
alert-comment-cc-users: '@ktrz'
5252
gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

‎.github/workflows/catch2_v2.yml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Catch2 C++ Example
2+
on:
3+
push:
4+
branches:
5+
- master
6+
7+
permissions:
8+
contents: write
9+
deployments: write
10+
11+
jobs:
12+
benchmark:
13+
name: Run Catch2 C++ Benchmark Framework example (v2.x)
14+
runs-on: ubuntu-20.04
15+
steps:
16+
- uses: actions/checkout@v4
17+
- name: Build and run benchmarks with Catch2
18+
run: |
19+
cd examples/catch2_v2
20+
mkdir build && cd build
21+
cmake -DCMAKE_BUILD_TYPE=Release ..
22+
cmake --build . --config Release
23+
./Catch2_bench | tee ../benchmark_result.txt
24+
25+
- name: Store benchmark result
26+
uses: benchmark-action/github-action-benchmark@v1
27+
with:
28+
name: Catch2 Benchmark
29+
tool: 'catch2'
30+
output-file-path: examples/catch2_v2/benchmark_result.txt
31+
github-token: ${{ secrets.GITHUB_TOKEN }}
32+
auto-push: true
33+
# Show alert with commit comment on detecting possible performance regression
34+
alert-threshold: '200%'
35+
comment-on-alert: true
36+
fail-on-alert: true
37+
alert-comment-cc-users: '@ktrz'
38+
39+
- name: Store benchmark result - separate results repo
40+
uses: benchmark-action/github-action-benchmark@v1
41+
with:
42+
name: Catch2 Benchmark
43+
tool: 'catch2'
44+
output-file-path: examples/catch2_v2/benchmark_result.txt
45+
github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
46+
auto-push: true
47+
# Show alert with commit comment on detecting possible performance regression
48+
alert-threshold: '200%'
49+
comment-on-alert: true
50+
fail-on-alert: true
51+
alert-comment-cc-users: '@ktrz'
52+
gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

‎.github/workflows/ci-results-repo.yml

+43-4
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,8 @@ jobs:
7979
github-token: ${{ secrets.GITHUB_TOKEN }}
8080
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository'
8181

82-
catch2-framework:
83-
name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
82+
catch2-v2-framework:
83+
name: Run Catch2 C++ Benchmark Framework example (v2.x) - github.com/benchmark-action/github-action-benchmark-results
8484
runs-on: ubuntu-20.04
8585
steps:
8686
- uses: actions/checkout@v4
@@ -92,7 +92,7 @@ jobs:
9292
- run: npm run build
9393
- name: Run benchmark
9494
run: |
95-
cd examples/catch2
95+
cd examples/catch2_v2
9696
mkdir build && cd build
9797
cmake -DCMAKE_BUILD_TYPE=Release ..
9898
cmake --build . --config Release
@@ -112,12 +112,51 @@ jobs:
112112
with:
113113
name: Catch2 Benchmark
114114
tool: 'catch2'
115-
output-file-path: examples/catch2/benchmark_result.txt
115+
output-file-path: examples/catch2_v2/benchmark_result.txt
116116
fail-on-alert: true
117117
gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
118118
github-token: ${{ secrets.GITHUB_TOKEN }}
119119
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository'
120120

121+
catch2-v3-framework:
122+
name: Run Catch2 C++ Benchmark Framework example (v3.x) - github.com/benchmark-action/github-action-benchmark-results
123+
runs-on: ubuntu-20.04
124+
steps:
125+
- uses: actions/checkout@v4
126+
- uses: actions/setup-node@v4
127+
with:
128+
node-version: 20
129+
cache: 'npm'
130+
- run: npm ci
131+
- run: npm run build
132+
- name: Run benchmark
133+
run: |
134+
cd examples/catch2
135+
mkdir build && cd build
136+
cmake -DCMAKE_BUILD_TYPE=Release ..
137+
cmake --build . --config Release
138+
./Catch2_bench > ../benchmark_result.txt
139+
140+
- uses: actions/checkout@v4
141+
with:
142+
repository: benchmark-action/github-action-benchmark-results
143+
ref: 'gh-pages'
144+
path: 'dist/other-repo'
145+
- name: Save previous data.js
146+
run: |
147+
cp ./dist/other-repo/dev/bench/data.js before_data.js
148+
149+
- name: Store benchmark result
150+
uses: ./
151+
with:
152+
name: Catch2 Benchmark (v3)
153+
tool: 'catch2'
154+
output-file-path: examples/catch2/benchmark_result.txt
155+
fail-on-alert: true
156+
gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
157+
github-token: ${{ secrets.GITHUB_TOKEN }}
158+
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' './benchmark-data-repository'
159+
121160
cpp-framework:
122161
name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
123162
runs-on: ubuntu-20.04

‎.github/workflows/ci.yml

+41-4
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ jobs:
7575
comment-on-alert: true
7676
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark'
7777

78-
catch2-framework:
79-
name: Run Catch2 C++ Benchmark Framework example
78+
catch2-v2-framework:
79+
name: Run Catch2 C++ Benchmark Framework example (v2.x)
8080
runs-on: ubuntu-20.04
8181
steps:
8282
- uses: actions/checkout@v4
@@ -88,7 +88,7 @@ jobs:
8888
- run: npm run build
8989
- name: Run benchmark
9090
run: |
91-
cd examples/catch2
91+
cd examples/catch2_v2
9292
mkdir build && cd build
9393
cmake -DCMAKE_BUILD_TYPE=Release ..
9494
cmake --build . --config Release
@@ -104,14 +104,51 @@ jobs:
104104
with:
105105
name: Catch2 Benchmark
106106
tool: 'catch2'
107-
output-file-path: examples/catch2/benchmark_result.txt
107+
output-file-path: examples/catch2_v2/benchmark_result.txt
108108
skip-fetch-gh-pages: true
109109
fail-on-alert: true
110110
summary-always: true
111111
github-token: ${{ secrets.GITHUB_TOKEN }}
112112
comment-on-alert: true
113113
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark'
114114

115+
catch2-v3-framework:
116+
name: Run Catch2 C++ Benchmark Framework example (v3.x)
117+
runs-on: ubuntu-20.04
118+
steps:
119+
- uses: actions/checkout@v4
120+
- uses: actions/setup-node@v4
121+
with:
122+
node-version: 20
123+
cache: 'npm'
124+
- run: npm ci
125+
- run: npm run build
126+
- name: Run benchmark
127+
run: |
128+
cd examples/catch2
129+
mkdir build && cd build
130+
cmake -DCMAKE_BUILD_TYPE=Release ..
131+
cmake --build . --config Release
132+
./Catch2_bench > ../benchmark_result.txt
133+
- name: Save previous data.js
134+
run: |
135+
git fetch origin gh-pages
136+
git checkout gh-pages
137+
cp ./dev/bench/data.js before_data.js
138+
git checkout -
139+
- name: Store benchmark result
140+
uses: ./
141+
with:
142+
name: Catch2 Benchmark (v3)
143+
tool: 'catch2'
144+
output-file-path: examples/catch2/benchmark_result.txt
145+
skip-fetch-gh-pages: true
146+
fail-on-alert: true
147+
summary-always: true
148+
github-token: ${{ secrets.GITHUB_TOKEN }}
149+
comment-on-alert: true
150+
- run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)'
151+
115152
cpp-framework:
116153
name: Run Google C++ Benchmark Framework example
117154
runs-on: ubuntu-20.04

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
## Unreleased
2+
- **fix** Catch2 v.3.5.0 changed output format [#241]
23

34
<a name="v1.20.2"></a>
45
# [v1.20.2](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.20.2) - 19 May 2024

‎examples/catch2/CMakeLists.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ include(FetchContent)
66
FetchContent_Declare(
77
catch2
88
GIT_REPOSITORY https://github.com/catchorg/Catch2.git
9-
GIT_TAG v2.11.0)
9+
GIT_TAG v3.6.0)
1010

1111
FetchContent_GetProperties(catch2)
1212
if(NOT catch2_POPULATED)
@@ -16,7 +16,7 @@ endif()
1616

1717
add_executable(${PROJECT_NAME})
1818
target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp)
19-
target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
19+
target_link_libraries(${PROJECT_NAME} Catch2::Catch2WithMain)
2020

2121
target_compile_options(
2222
${PROJECT_NAME}

‎examples/catch2/catch2_bench.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#include "fib.hpp"
2-
#define CATCH_CONFIG_MAIN
3-
#include <catch2/catch.hpp>
2+
#include <catch2/catch_test_macros.hpp>
3+
#include <catch2/benchmark/catch_benchmark.hpp>
44

55
TEST_CASE("Fibonacci") {
66

‎examples/catch2_v2/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
build/

‎examples/catch2_v2/CMakeLists.txt

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
cmake_minimum_required(VERSION 3.12)
2+
project("Catch2_bench")
3+
4+
include(FetchContent)
5+
6+
FetchContent_Declare(
7+
catch2
8+
GIT_REPOSITORY https://github.com/catchorg/Catch2.git
9+
GIT_TAG v2.13.10)
10+
11+
FetchContent_GetProperties(catch2)
12+
if(NOT catch2_POPULATED)
13+
FetchContent_Populate(catch2)
14+
add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR})
15+
endif()
16+
17+
add_executable(${PROJECT_NAME})
18+
target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp)
19+
target_link_libraries(${PROJECT_NAME} Catch2::Catch2)
20+
21+
target_compile_options(
22+
${PROJECT_NAME}
23+
PRIVATE
24+
$<$<CXX_COMPILER_ID:MSVC>:/DCATCH_CONFIG_ENABLE_BENCHMARKING>
25+
$<$<OR:$<CXX_COMPILER_ID:GNU>,$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>>:-DCATCH_CONFIG_ENABLE_BENCHMARKING>
26+
)

‎examples/catch2_v2/README.md

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
C++ example for benchmarking with [Catch2 Framework][tool]
2+
====================================================================
3+
4+
- [Workflow for this example](../../.github/workflows/catch2.yml)
5+
- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22)
6+
- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/)
7+
8+
This directory shows how to use [`github-action-benchmark`][action] with [Catch2 Framework][tool].
9+
10+
11+
12+
## Run benchmarks
13+
14+
Official documentation for usage of Catch2 Framework can be found in its repository:
15+
16+
https://github.com/catchorg/Catch2
17+
18+
Since Catch2 is a header-only test framework, you don't need to build it in advance.
19+
Download and put the headers in your `include` directory and write your benchmarks.
20+
21+
```cpp
22+
#define CATCH_CONFIG_MAIN
23+
#include <catch2/catch.hpp>
24+
25+
TEST_CASE("Fibonacci") {
26+
// now let's benchmark:
27+
BENCHMARK("Some benchmark") {
28+
// Your benchmark goes here
29+
};
30+
}
31+
```
32+
33+
Build the source with C++ compiler and run the built executable to get the benchmark output.
34+
Ensure to use `console` reporter for this. `xml` reporter may be supported in the future.
35+
36+
37+
38+
## Process benchmark results
39+
40+
Store the benchmark results with step using the action. Please set `catch2` to `tool` input.
41+
42+
```yaml
43+
- name: Store benchmark result
44+
uses: benchmark-action/github-action-benchmark@v1
45+
with:
46+
tool: 'catch2'
47+
output-file-path: benchmark_result.json
48+
```
49+
50+
Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage.
51+
52+
53+
54+
## Run this example
55+
56+
To try this example, please use [cmake](./CMakeLists.txt) and `clang++`.
57+
58+
```sh
59+
$ mkdir build
60+
$ cd build
61+
$ cmake -DCMAKE_BUILD_TYPE=Release ..
62+
$ cmake --build . --config Release
63+
```
64+
65+
This will create `Catch2_bench` executable. The results are output to stdout.
66+
67+
[tool]: https://github.com/catchorg/Catch2
68+
[action]: https://github.com/benchmark-action/github-action-benchmark

‎examples/catch2_v2/catch2_bench.cpp

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#include "fib.hpp"
2+
#define CATCH_CONFIG_MAIN
3+
#include <catch2/catch.hpp>
4+
5+
TEST_CASE("Fibonacci") {
6+
7+
// now let's benchmark:
8+
BENCHMARK("Fibonacci 10") { return fib(10); };
9+
10+
BENCHMARK("Fibonacci 20") { return fib(20); };
11+
}

‎examples/catch2_v2/fib.hpp

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#if !defined FIB_HPP_INCLUDED
2+
#define FIB_HPP_INCLUDED
3+
4+
int fib(int const i) {
5+
if (i <= 1) {
6+
return 1;
7+
}
8+
return fib(i - 2) + fib(i - 1);
9+
}
10+
11+
#endif // FIB_HPP_INCLUDED

0 commit comments

Comments
 (0)