@@ -14,7 +14,71 @@ The following is an example of a popular Vector configuration that ingests logs
14
14
from a file and routes them to both Elasticsearch and AWS S3. Your configuration
15
15
will differ based on your needs.
16
16
17
- {{< tabs default="vector.toml" >}}
17
+ {{< tabs default="vector.yaml" >}}
18
+ {{< tab title="vector.yaml" >}}
19
+
20
+ ``` yaml
21
+ # Set global options
22
+ data_dir : " /var/lib/vector"
23
+
24
+ # Vector's API (disabled by default)
25
+ # Enable and try it out with the `vector top` command
26
+ api :
27
+ enabled : false
28
+ # address = "127.0.0.1:8686"
29
+
30
+ # Ingest data by tailing one or more files
31
+ sources :
32
+ apache_logs :
33
+ type : " file"
34
+ include :
35
+ - " /var/log/apache2/*.log" # supports globbing
36
+ ignore_older : 86400 # 1 day
37
+
38
+ # Structure and parse via Vector's Remap Language
39
+ transforms :
40
+ apache_parser :
41
+ inputs :
42
+ - " apache_logs"
43
+ type : " remap"
44
+ source : " . = parse_apache_log(.message)"
45
+
46
+ # Sample the data to save on cost
47
+ apache_sampler :
48
+ inputs :
49
+ - " apache_parser"
50
+ type : " sample"
51
+ rate : 2 # only keep 50% (1/`rate`)
52
+
53
+ # Send structured data to a short-term storage
54
+ sinks :
55
+ es_cluster :
56
+ inputs :
57
+ - " apache_sampler" # only take sampled data
58
+ type : " elasticsearch"
59
+ endpoints :
60
+ - " http://79.12.221.222:9200"
61
+ bulk :
62
+ index : " vector-%Y-%m-%d" # daily indices
63
+
64
+ # Send structured data to a cost-effective long-term storage
65
+ s3_archives :
66
+ inputs :
67
+ - " apache_parser" # don't sample for S3
68
+ type : " aws_s3"
69
+ region : " us-east-1"
70
+ bucket : " my-log-archives"
71
+ key_prefix : " date=%Y-%m-%d" # daily partitions, hive friendly format
72
+ compression : " gzip" # compress final objects
73
+ framing :
74
+ method : " newline_delimited" # new line delimited...
75
+ encoding :
76
+ codec : " json" # ...JSON
77
+ batch :
78
+ max_bytes : 10000000 # 10mb uncompressed
79
+ ` ` `
80
+
81
+ {{< /tab >}}
18
82
{{< tab title="vector.toml" >}}
19
83
20
84
` ` ` toml
@@ -67,53 +131,6 @@ encoding.codec = "json" # ...JSON
67
131
batch.max_bytes = 10000000 # 10mb uncompressed
68
132
```
69
133
70
- {{< /tab >}}
71
- {{< tab title="vector.yaml" >}}
72
-
73
- ``` yaml
74
- data_dir : /var/lib/vector
75
- sources :
76
- apache_logs :
77
- type : file
78
- include :
79
- - /var/log/apache2/*.log
80
- ignore_older : 86400
81
- transforms :
82
- apache_parser :
83
- inputs :
84
- - apache_logs
85
- type : remap
86
- source : |
87
- . = parse_apache_log(.message)
88
- apache_sampler :
89
- inputs :
90
- - apache_parser
91
- type : sample
92
- rate : 50
93
- sinks :
94
- es_cluster :
95
- inputs :
96
- - apache_sampler
97
- type : elasticsearch
98
- endpoints : ['http://79.12.221.222:9200']
99
- bulk :
100
- index : vector-%Y-%m-%d
101
- s3_archives :
102
- inputs :
103
- - apache_parser
104
- type : aws_s3
105
- region : us-east-1
106
- bucket : my-log-archives
107
- key_prefix : date=%Y-%m-%d
108
- compression : gzip
109
- framing :
110
- method : newline_delimited
111
- encoding :
112
- codec : json
113
- batch :
114
- max_bytes : 10000000
115
- ` ` `
116
-
117
134
{{< /tab >}}
118
135
{{< tab title="vector.json" >}}
119
136
@@ -185,18 +202,18 @@ sinks:
185
202
To use this configuration file, specify it with the ` --config ` flag when
186
203
starting Vector:
187
204
188
- {{< tabs default="TOML " >}}
189
- {{< tab title="TOML " >}}
205
+ {{< tabs default="YAML " >}}
206
+ {{< tab title="YAML " >}}
190
207
191
208
``` shell
192
- vector --config /etc/vector/vector.toml
209
+ vector --config /etc/vector/vector.yaml
193
210
```
194
211
195
212
{{< /tab >}}
196
- {{< tab title="YAML " >}}
213
+ {{< tab title="TOML " >}}
197
214
198
215
``` shell
199
- vector --config /etc/vector/vector.yaml
216
+ vector --config /etc/vector/vector.toml
200
217
```
201
218
202
219
{{< /tab >}}
@@ -229,37 +246,37 @@ vector --config /etc/vector/vector.json
229
246
Vector interpolates environment variables within your configuration file with
230
247
the following syntax:
231
248
232
- ``` toml
233
- [transforms .add_host ]
234
- type = " remap"
235
- source = '''
236
- # Basic usage. "$HOSTNAME" also works.
237
- .host = "${HOSTNAME}" # or "$HOSTNAME"
249
+ ``` yaml
250
+ transforms :
251
+ add_host :
252
+ type : " remap"
253
+ source : |
254
+ # Basic usage. "$HOSTNAME" also works.
255
+ .host = "${HOSTNAME}" # or "$HOSTNAME"
238
256
239
- # Setting a default value when not present.
240
- .environment = "${ENV:-development}"
257
+ # Setting a default value when not present.
258
+ .environment = "${ENV:-development}"
241
259
242
- # Requiring an environment variable to be present.
243
- .tenant = "${TENANT:?tenant must be supplied}"
244
- '''
260
+ # Requiring an environment variable to be present.
261
+ .tenant = "${TENANT:?tenant must be supplied}"
245
262
` ` `
246
263
247
264
#### Default values
248
265
249
266
Default values can be supplied using ` :-` or `-` syntax:
250
267
251
- ``` toml
252
- option = " ${ENV_VAR:-default}" # default value if variable is unset or empty
253
- option = " ${ENV_VAR-default}" # default value only if variable is unset
268
+ ` ` ` yaml
269
+ option: "${ENV_VAR:-default}" # default value if variable is unset or empty
270
+ option: "${ENV_VAR-default}" # default value only if variable is unset
254
271
` ` `
255
272
256
273
# ### Required variables
257
274
258
275
Environment variables that are required can be specified using `:?` or `?` syntax :
259
276
260
- ``` toml
261
- option = " ${ENV_VAR:?err}" # Vector exits with 'err' message if variable is unset or empty
262
- option = " ${ENV_VAR?err}" # Vector exits with 'err' message only if variable is unset
277
+ ` ` ` yaml
278
+ option: "${ENV_VAR:?err}" # Vector exits with 'err' message if variable is unset or empty
279
+ option: "${ENV_VAR?err}" # Vector exits with 'err' message only if variable is unset
263
280
` ` `
264
281
265
282
# ### Escaping
@@ -285,89 +302,98 @@ method. For most Linux-based systems, the file can be found at
285
302
You can pass multiple configuration files when starting Vector :
286
303
287
304
` ` ` shell
288
- vector --config vector1.toml --config vector2.toml
305
+ vector --config vector1.yaml --config vector2.yaml
289
306
` ` `
290
307
291
308
Or using a [globbing syntax][glob] :
292
309
293
310
` ` ` shell
294
- vector --config /etc/vector/* .toml
311
+ vector --config /etc/vector/*.yaml
295
312
` ` `
296
313
297
314
# ### Automatic namespacing
298
315
299
316
You can also split your configuration by grouping the components by their type, one directory per component type, where the file name is used as the component id. For example :
300
317
301
- {{< tabs default="vector.toml " >}}
302
- {{< tab title="vector.toml " >}}
318
+ {{< tabs default="vector.yaml " >}}
319
+ {{< tab title="vector.yaml " >}}
303
320
304
- ``` toml
321
+ ` ` ` yaml
305
322
# Set global options
306
- data_dir = " /var/lib/vector"
323
+ data_dir: "/var/lib/vector"
307
324
308
325
# Vector's API (disabled by default)
309
326
# Enable and try it out with the ` vector top` command
310
- [ api ]
311
- enabled = false
312
- # address = "127.0.0.1:8686"
327
+ api :
328
+ enabled : false
329
+ # address: "127.0.0.1:8686"
313
330
```
314
331
315
332
{{< /tab >}}
316
- {{< tab title="sources/apache_logs.toml " >}}
333
+ {{< tab title="sources/apache_logs.yaml " >}}
317
334
318
- ``` toml
335
+ ``` yaml
319
336
# Ingest data by tailing one or more files
320
- type = " file"
321
- include = [" /var/log/apache2/*.log" ] # supports globbing
322
- ignore_older = 86400 # 1 day
337
+ type : " file"
338
+ include : ["/var/log/apache2/*.log"] # supports globbing
339
+ ignore_older : 86400 # 1 day
323
340
` ` `
324
341
325
342
{{< /tab >}}
326
- {{< tab title="transforms/apache_parser.toml " >}}
343
+ {{< tab title="transforms/apache_parser.yaml " >}}
327
344
328
- ``` toml
345
+ ` ` ` yaml
329
346
# Structure and parse via Vector Remap Language
330
- inputs = [" apache_logs" ]
331
- type = " remap"
332
- source = '''
333
- . = parse_apache_log(.message)
347
+ inputs :
348
+ - " apache_logs"
349
+ type : " remap"
350
+ source : |
351
+ . = parse_apache_log(.message)
334
352
` ` `
335
353
336
354
{{< /tab >}}
337
- {{< tab title="transforms/apache_sampler.toml " >}}
355
+ {{< tab title="transforms/apache_sampler.yaml " >}}
338
356
339
- ```toml
357
+ ` ` ` yaml
340
358
# Sample the data to save on cost
341
- inputs = ["apache_parser"]
342
- type = "sample"
343
- rate = 2 # only keep 50% (1/`rate`)
359
+ inputs :
360
+ - " apache_parser"
361
+ type : " sample"
362
+ rate : 2 # only keep 50% (1/`rate`)
344
363
` ` `
345
364
346
365
{{< /tab >}}
347
- {{< tab title="sinks/es_cluster.toml " >}}
366
+ {{< tab title="sinks/es_cluster.yaml " >}}
348
367
349
- ```toml
368
+ ` ` ` yaml
350
369
# Send structured data to a short-term storage
351
- inputs = ["apache_sampler"] # only take sampled data
352
- type = "elasticsearch"
353
- endpoints = ["http://79.12.221.222:9200"] # local or external host
354
- bulk.index = "vector-%Y-%m-%d" # daily indices
370
+ inputs :
371
+ - " apache_sampler" # only take sampled data
372
+ type : " elasticsearch"
373
+ endpoints :
374
+ - " http://79.12.221.222:9200" # local or external host
375
+ bulk :
376
+ index : " vector-%Y-%m-%d" # daily indices
355
377
` ` `
356
378
357
379
{{< /tab >}}
358
- {{< tab title="sinks/s3_archives.toml " >}}
380
+ {{< tab title="sinks/s3_archives.yaml " >}}
359
381
360
- ```toml
382
+ ` ` ` yaml
361
383
# Send structured data to a cost-effective long-term storage
362
- inputs = ["apache_parser"] # don't sample for S3
363
- type = "aws_s3"
364
- region = "us-east-1"
365
- bucket = "my-log-archives"
366
- key_prefix = "date=%Y-%m-%d" # daily partitions, hive friendly format
367
- compression = "gzip" # compress final objects
368
- framing.method = "newline_delimited" # new line delimited...
369
- encoding.codec = "json" # ...JSON
370
- batch.max_bytes = 10000000 # 10mb uncompressed
384
+ inputs :
385
+ - " apache_parser" # don't sample for S3
386
+ type : " aws_s3"
387
+ region : " us-east-1"
388
+ bucket : " my-log-archives"
389
+ key_prefix : " date=%Y-%m-%d" # daily partitions, hive-friendly format
390
+ compression : " gzip" # compress final objects
391
+ framing :
392
+ method : " newline_delimited" # new line delimited...
393
+ encoding :
394
+ codec : " json" # ...JSON
395
+ batch :
396
+ max_bytes : 10000000 # 10mb uncompressed
371
397
` ` `
372
398
373
399
{{< /tab >}}
@@ -384,26 +410,28 @@ vector --config-dir /etc/vector
384
410
Vector supports wildcards (`*`) in component IDs when building your topology.
385
411
For example :
386
412
387
- ```toml
388
- [sources.app1_logs]
389
- type = "file"
390
- includes = ["/var/log/app1.log"]
413
+ ` ` ` yaml
414
+ sources:
415
+ app1_logs:
416
+ type: "file"
417
+ includes: ["/var/log/app1.log"]
391
418
392
- [sources. app2_logs]
393
- type = "file"
394
- includes = ["/var/log/app.log"]
419
+ app2_logs:
420
+ type: "file"
421
+ includes: ["/var/log/app.log"]
395
422
396
- [sources. system_logs]
397
- type = "file"
398
- includes = ["/var/log/system.log"]
423
+ system_logs:
424
+ type: "file"
425
+ includes: ["/var/log/system.log"]
399
426
400
- [sinks.app_logs]
401
- type = "datadog_logs"
402
- inputs = ["app*"]
427
+ sinks:
428
+ app_logs:
429
+ type: "datadog_logs"
430
+ inputs: ["app*"]
403
431
404
- [sinks. archive]
405
- type = "aws_s3"
406
- inputs = ["app*", "system_logs"]
432
+ archive:
433
+ type: "aws_s3"
434
+ inputs: ["app*", "system_logs"]
407
435
` ` `
408
436
409
437
# # Sections
0 commit comments