Skip to content

Commit 060399a

Browse files
neuronullneuronull
authored
chore(ci): Move most CI checks to merge queue (vectordotdev#17340)
Introduces a number of changes to our project's GH workflows: - There is a more limited set of tests that run on each PR commit, and the rest were moved to the merge queue. - Most workflows can be triggered manually by a team member though a comment on a PR. - Integration tests are run on PR commits if they touch the files specific to that component.
1 parent 618379a commit 060399a

21 files changed

+2308
-1101
lines changed

‎.github/workflows/changes.yml

+333
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
# This workflow identifies changes between the base and the head ref, for use in
2+
# other workflows to decide if they should be executed.
3+
4+
name: Identify Changes
5+
6+
on:
7+
workflow_call:
8+
# These inputs allow the filter action to be able to access the correct refs for
9+
# comparison in changes detection, it is required as this is called from the
10+
# merge_group context.
11+
inputs:
12+
base_ref:
13+
required: true
14+
type: string
15+
head_ref:
16+
required: true
17+
type: string
18+
outputs:
19+
source:
20+
value: ${{ jobs.changes.outputs.source }}
21+
dependencies:
22+
value: ${{ jobs.changes.outputs.dependencies }}
23+
internal_events:
24+
value: ${{ jobs.changes.outputs.internal_events }}
25+
cue:
26+
value: ${{ jobs.changes.outputs.cue }}
27+
component_docs:
28+
value: ${{ jobs.changes.outputs.component_docs }}
29+
markdown:
30+
value: ${{ jobs.changes.outputs.markdown }}
31+
install:
32+
value: ${{ jobs.changes.outputs.install }}
33+
k8s:
34+
value: ${{ jobs.changes.outputs.k8s }}
35+
all-int:
36+
value: ${{ jobs.changes.outputs.all-int }}
37+
amqp:
38+
value: ${{ jobs.changes.outputs.amqp }}
39+
appsignal:
40+
value: ${{ jobs.changes.outputs.appsignal }}
41+
aws:
42+
value: ${{ jobs.changes.outputs.aws }}
43+
axiom:
44+
value: ${{ jobs.changes.outputs.axiom }}
45+
azure:
46+
value: ${{ jobs.changes.outputs.azure }}
47+
clickhouse:
48+
value: ${{ jobs.changes.outputs.clickhouse }}
49+
databend:
50+
value: ${{ jobs.changes.outputs.databend }}
51+
datadog:
52+
value: ${{ jobs.changes.outputs.datadog }}
53+
dnstap:
54+
value: ${{ jobs.changes.outputs.dnstap }}
55+
docker-logs:
56+
value: ${{ jobs.changes.outputs.docker-logs }}
57+
elasticsearch:
58+
value: ${{ jobs.changes.outputs.elasticsearch }}
59+
eventstoredb:
60+
value: ${{ jobs.changes.outputs.eventstoredb }}
61+
fluent:
62+
value: ${{ jobs.changes.outputs.fluent }}
63+
gcp:
64+
value: ${{ jobs.changes.outputs.gcp }}
65+
humio:
66+
value: ${{ jobs.changes.outputs.humio }}
67+
http-client:
68+
value: ${{ jobs.changes.outputs.http-client }}
69+
influxdb:
70+
value: ${{ jobs.changes.outputs.influxdb }}
71+
kafka:
72+
value: ${{ jobs.changes.outputs.kafka }}
73+
logstash:
74+
value: ${{ jobs.changes.outputs.logstash }}
75+
loki:
76+
value: ${{ jobs.changes.outputs.loki }}
77+
mongodb:
78+
value: ${{ jobs.changes.outputs.mongodb }}
79+
nats:
80+
value: ${{ jobs.changes.outputs.nats }}
81+
nginx:
82+
value: ${{ jobs.changes.outputs.nginx }}
83+
opentelemetry:
84+
value: ${{ jobs.changes.outputs.opentelemetry }}
85+
postgres:
86+
value: ${{ jobs.changes.outputs.postgres }}
87+
prometheus:
88+
value: ${{ jobs.changes.outputs.prometheus }}
89+
pulsar:
90+
value: ${{ jobs.changes.outputs.pulsar }}
91+
redis:
92+
value: ${{ jobs.changes.outputs.redis }}
93+
splunk:
94+
value: ${{ jobs.changes.outputs.splunk }}
95+
webhdfs:
96+
value: ${{ jobs.changes.outputs.webhdfs }}
97+
98+
jobs:
99+
changes:
100+
runs-on: ubuntu-20.04
101+
# Set job outputs to values from filter step
102+
outputs:
103+
# General source code
104+
source: ${{ steps.filter.outputs.source }}
105+
dependencies: ${{ steps.filter.outputs.dependencies }}
106+
internal_events: ${{ steps.filter.outputs.internal_events }}
107+
cue: ${{ steps.filter.outputs.cue }}
108+
component_docs: ${{ steps.filter.outputs.component_docs }}
109+
markdown: ${{ steps.filter.outputs.markdown }}
110+
install: ${{ steps.filter.outputs.install }}
111+
# K8s
112+
k8s: ${{ steps.filter.outputs.k8s }}
113+
# Integrations
114+
all-int: ${{ steps.filter.outputs.all-int }}
115+
amqp: ${{ steps.filter.outputs.amqp }}
116+
appsignal: ${{ steps.filter.outputs.appsignal}}
117+
aws: ${{ steps.filter.outputs.aws }}
118+
axiom: ${{ steps.filter.outputs.axiom }}
119+
azure: ${{ steps.filter.outputs.azure }}
120+
clickhouse: ${{ steps.filter.outputs.clickhouse }}
121+
databend: ${{ steps.filter.outputs.databend }}
122+
datadog: ${{ steps.filter.outputs.datadog }}
123+
dnstap: ${{ steps.filter.outputs.dnstap }}
124+
docker-logs: ${{ steps.filter.outputs.docker-logs }}
125+
elasticsearch: ${{ steps.filter.outputs.elasticsearch }}
126+
eventstoredb: ${{ steps.filter.outputs.eventstoredb }}
127+
fluent: ${{ steps.filter.outputs.fluent }}
128+
gcp: ${{ steps.filter.outputs.gcp }}
129+
humio: ${{ steps.filter.outputs.humio }}
130+
http-client: ${{ steps.filter.outputs.http-client }}
131+
influxdb: ${{ steps.filter.outputs.influxdb }}
132+
kafka: ${{ steps.filter.outputs.kafka }}
133+
logstash: ${{ steps.filter.outputs.logstash }}
134+
loki: ${{ steps.filter.outputs.loki }}
135+
mongodb: ${{ steps.filter.outputs.mongodb }}
136+
nats: ${{ steps.filter.outputs.nats }}
137+
nginx: ${{ steps.filter.outputs.nginx }}
138+
opentelemetry: ${{ steps.filter.outputs.opentelemetry }}
139+
postgres: ${{ steps.filter.outputs.postgres }}
140+
prometheus: ${{ steps.filter.outputs.prometheus }}
141+
pulsar: ${{ steps.filter.outputs.pulsar }}
142+
redis: ${{ steps.filter.outputs.redis }}
143+
splunk: ${{ steps.filter.outputs.splunk }}
144+
webhdfs: ${{ steps.filter.outputs.webhdfs }}
145+
steps:
146+
- uses: actions/checkout@v3
147+
148+
- uses: dorny/paths-filter@v2
149+
id: filter
150+
with:
151+
base: ${{ inputs.base_ref }}
152+
ref: ${{ inputs.head_ref }}
153+
filters: |
154+
source:
155+
- ".github/workflows/test.yml"
156+
- ".cargo/**"
157+
- "benches/**"
158+
- "lib/**"
159+
- "proto/**"
160+
- "scripts/**"
161+
- "src/**"
162+
- "tests/**"
163+
- "build.rs"
164+
- "Cargo.lock"
165+
- "Cargo.toml"
166+
- "Makefile"
167+
- "rust-toolchain.toml"
168+
- "vdev/**"
169+
deny:
170+
- 'deny.toml'
171+
- "vdev/**"
172+
dependencies:
173+
- ".cargo/**"
174+
- 'Cargo.toml'
175+
- 'Cargo.lock'
176+
- 'rust-toolchain.toml'
177+
- '.github/workflows/pr.yml'
178+
- 'Makefile'
179+
- 'scripts/cross/**'
180+
- "vdev/**"
181+
cue:
182+
- 'website/cue/**'
183+
- "vdev"
184+
component_docs:
185+
- 'scripts/generate-component-docs.rb'
186+
- "vdev/**"
187+
markdown:
188+
- '**/**.md'
189+
- "vdev/**"
190+
internal_events:
191+
- 'src/internal_events/**'
192+
- "vdev/**"
193+
docker:
194+
- 'distribution/docker/**'
195+
- "vdev/**"
196+
install:
197+
- ".github/workflows/install-sh.yml"
198+
- "distribution/install.sh"
199+
k8s:
200+
- "src/sources/kubernetes_logs/**"
201+
all-int:
202+
- "lib/vector-core/**"
203+
amqp:
204+
- "src/amqp.rs"
205+
- "src/internal_events/amqp.rs"
206+
- "src/sinks/amqp/**"
207+
- "src/sources/amqp.rs"
208+
- "src/sources/util/**"
209+
- "src/sinks/util/**"
210+
appsignal:
211+
- "src/sinks/appsignal/**"
212+
- "src/sinks/util/**"
213+
aws:
214+
- "src/aws_**"
215+
- "src/internal_events/aws_**"
216+
- "src/sources/aws_**"
217+
- "src/sources/util/**"
218+
- "src/sinks/aws_**"
219+
- "src/sinks/util/**"
220+
- "src/transforms/aws_**"
221+
axiom:
222+
- "src/sinks/axiom.rs"
223+
- "src/sinks/util/**"
224+
azure:
225+
- "src/sinks/azure_**"
226+
- "src/sinks/util/**"
227+
clickhouse:
228+
- "src/sinks/clickhouse/**"
229+
- "src/sinks/util/**"
230+
databend:
231+
- "src/sinks/databend/**"
232+
- "src/sinks/util/**"
233+
datadog:
234+
- "src/common/datadog.rs"
235+
- "src/internal_events/datadog_*"
236+
- "src/sources/datadog_agent/**"
237+
- "src/sinks/datadog/**"
238+
- "src/sinks/datadog_archives.rs"
239+
- "src/sinks/util/**"
240+
docker-logs:
241+
- "src/docker.rs"
242+
- "src/internal_events/docker_logs.rs"
243+
- "src/sources/docker_logs/**"
244+
- "src/sources/util/**"
245+
elasticsearch:
246+
- "src/sinks/elasticsearch/**"
247+
- "src/sinks/util/**"
248+
eventstoredb:
249+
- "src/internal_events/eventstoredb_metrics.rs"
250+
- "src/sources/eventstoredb_metrics/**"
251+
- "src/sources/util/**"
252+
fluent:
253+
- "src/internal_events/fluent.rs"
254+
- "src/sources/fluent/**"
255+
- "src/sources/util/**"
256+
gcp:
257+
- "src/internal_events/gcp_pubsub.rs"
258+
- "src/sources/gcp_pubsub.rs"
259+
- "src/sources/util/**"
260+
- "src/sinks/gcp/**"
261+
- "src/sinks/util/**"
262+
- "src/gcp.rs"
263+
humio:
264+
- "src/sinks/humio/**"
265+
- "src/sinks/util/**"
266+
http-client:
267+
- "src/sinks/http-client/**"
268+
influxdb:
269+
- "src/internal_events/influxdb.rs"
270+
- "src/sinks/influxdb/**"
271+
- "src/sinks/util/**"
272+
kafka:
273+
- "src/internal_events/kafka.rs"
274+
- "src/sinks/kafka/**"
275+
- "src/sinks/util/**"
276+
- "src/sources/kafka.rs"
277+
- "src/sources/util/**"
278+
- "src/kafka.rs"
279+
logstash:
280+
- "src/sources/logstash.rs"
281+
- "src/sources/util/**"
282+
loki:
283+
- "src/internal_events/loki.rs"
284+
- "src/sinks/loki/**"
285+
- "src/sinks/util/**"
286+
mongodb:
287+
- "src/internal_events/mongodb_metrics.rs"
288+
- "src/sources/mongodb_metrics/**"
289+
- "src/sources/util/**"
290+
nats:
291+
- "src/internal_events/nats.rs"
292+
- "src/sources/nats.rs"
293+
- "src/sources/util/**"
294+
- "src/sinks/nats.rs"
295+
- "src/sinks/util/**"
296+
- "src/nats.rs"
297+
nginx:
298+
- "src/internal_events/nginx_metrics.rs"
299+
- "src/sources/nginx_metrics/**"
300+
- "src/sources/util/**"
301+
opentelemetry:
302+
- "src/sources/opentelemetry/**"
303+
- "src/sources/util/**"
304+
postgres:
305+
- "src/internal_events/postgresql_metrics.rs"
306+
- "src/sources/postgresql_metrics.rs"
307+
- "src/sources/util/**"
308+
prometheus:
309+
- "src/internal_events/prometheus.rs"
310+
- "src/sources/prometheus/**"
311+
- "src/sources/util/**"
312+
- "src/sinks/prometheus/**"
313+
- "src/sinks/util/**"
314+
pulsar:
315+
- "src/internal_events/pulsar.rs"
316+
- "src/sinks/pulsar/**"
317+
- "src/sinks/util/**"
318+
redis:
319+
- "src/internal_events/redis.rs"
320+
- "src/sources/redis/**"
321+
- "src/sources/util/**"
322+
- "src/sinks/redis.rs"
323+
- "src/sinks/util/**"
324+
splunk:
325+
- "src/internal_events/splunk_hec.rs"
326+
- "src/sources/splunk_hec/**"
327+
- "src/sources/util/**"
328+
- "src/sinks/splunk_hec/**"
329+
- "src/sinks/util/**"
330+
webhdfs:
331+
- "src/sinks/webhdfs/**"
332+
- "src/sinks/util/**"
333+

‎.github/workflows/cli.yml

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: CLI - Linux
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
test-cli:
8+
runs-on: [linux, ubuntu-20.04-8core]
9+
env:
10+
CARGO_INCREMENTAL: 0
11+
steps:
12+
- name: (PR comment) Get PR branch
13+
if: ${{ github.event_name == 'issue_comment' }}
14+
uses: xt0rted/pull-request-comment-branch@v1
15+
id: comment-branch
16+
17+
- name: (PR comment) Set latest commit status as pending
18+
if: ${{ github.event_name == 'issue_comment' }}
19+
uses: myrotvorets/[email protected]
20+
with:
21+
sha: ${{ steps.comment-branch.outputs.head_sha }}
22+
token: ${{ secrets.GITHUB_TOKEN }}
23+
context: CLI - Linux
24+
status: pending
25+
26+
- name: (PR comment) Checkout PR branch
27+
if: ${{ github.event_name == 'issue_comment' }}
28+
uses: actions/checkout@v3
29+
with:
30+
ref: ${{ steps.comment-branch.outputs.head_ref }}
31+
32+
- name: Checkout branch
33+
if: ${{ github.event_name != 'issue_comment' }}
34+
uses: actions/checkout@v3
35+
36+
- name: Cache Cargo registry + index
37+
uses: actions/cache@v3
38+
with:
39+
path: |
40+
~/.cargo/bin/
41+
~/.cargo/registry/index/
42+
~/.cargo/registry/cache/
43+
~/.cargo/git/db/
44+
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
45+
restore-keys: |
46+
${{ runner.os }}-cargo-
47+
48+
- run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
49+
- run: bash scripts/environment/prepare.sh
50+
- run: echo "::add-matcher::.github/matchers/rust.json"
51+
- run: make test-cli
52+
- name: Upload test results
53+
run: scripts/upload-test-results.sh
54+
if: always()
55+
56+
- name: (PR comment) Set latest commit status as ${{ job.status }}
57+
uses: myrotvorets/[email protected]
58+
if: always() && github.event_name == 'issue_comment'
59+
with:
60+
sha: ${{ steps.comment-branch.outputs.head_sha }}
61+
token: ${{ secrets.GITHUB_TOKEN }}
62+
context: CLI - Linux
63+
status: ${{ job.status }}

0 commit comments

Comments
 (0)