@@ -14,7 +14,71 @@ The following is an example of a popular Vector configuration that ingests logs
14
14
from a file and routes them to both Elasticsearch and AWS S3. Your configuration
15
15
will differ based on your needs.
16
16
17
- {{< tabs default="vector.toml" >}}
17
+ {{< tabs default="vector.yaml" >}}
18
+ {{< tab title="vector.yaml" >}}
19
+
20
+ ``` yaml
21
+ # Set global options
22
+ data_dir : " /var/lib/vector"
23
+
24
+ # Vector's API (disabled by default)
25
+ # Enable and try it out with the `vector top` command
26
+ api :
27
+ enabled : false
28
+ # address = "127.0.0.1:8686"
29
+
30
+ # Ingest data by tailing one or more files
31
+ sources :
32
+ apache_logs :
33
+ type : " file"
34
+ include :
35
+ - " /var/log/apache2/*.log" # supports globbing
36
+ ignore_older : 86400 # 1 day
37
+
38
+ # Structure and parse via Vector's Remap Language
39
+ transforms :
40
+ apache_parser :
41
+ inputs :
42
+ - " apache_logs"
43
+ type : " remap"
44
+ source : " . = parse_apache_log(.message)"
45
+
46
+ # Sample the data to save on cost
47
+ apache_sampler :
48
+ inputs :
49
+ - " apache_parser"
50
+ type : " sample"
51
+ rate : 2 # only keep 50% (1/`rate`)
52
+
53
+ # Send structured data to a short-term storage
54
+ sinks :
55
+ es_cluster :
56
+ inputs :
57
+ - " apache_sampler" # only take sampled data
58
+ type : " elasticsearch"
59
+ endpoints :
60
+ - " http://79.12.221.222:9200"
61
+ bulk :
62
+ index : " vector-%Y-%m-%d" # daily indices
63
+
64
+ # Send structured data to a cost-effective long-term storage
65
+ s3_archives :
66
+ inputs :
67
+ - " apache_parser" # don't sample for S3
68
+ type : " aws_s3"
69
+ region : " us-east-1"
70
+ bucket : " my-log-archives"
71
+ key_prefix : " date=%Y-%m-%d" # daily partitions, hive friendly format
72
+ compression : " gzip" # compress final objects
73
+ framing :
74
+ method : " newline_delimited" # new line delimited...
75
+ encoding :
76
+ codec : " json" # ...JSON
77
+ batch :
78
+ max_bytes : 10000000 # 10mb uncompressed
79
+ ` ` `
80
+
81
+ {{< /tab >}}
18
82
{{< tab title="vector.toml" >}}
19
83
20
84
` ` ` toml
@@ -67,53 +131,6 @@ encoding.codec = "json" # ...JSON
67
131
batch.max_bytes = 10000000 # 10mb uncompressed
68
132
```
69
133
70
- {{< /tab >}}
71
- {{< tab title="vector.yaml" >}}
72
-
73
- ``` yaml
74
- data_dir : /var/lib/vector
75
- sources :
76
- apache_logs :
77
- type : file
78
- include :
79
- - /var/log/apache2/*.log
80
- ignore_older : 86400
81
- transforms :
82
- apache_parser :
83
- inputs :
84
- - apache_logs
85
- type : remap
86
- source : |
87
- . = parse_apache_log(.message)
88
- apache_sampler :
89
- inputs :
90
- - apache_parser
91
- type : sample
92
- rate : 50
93
- sinks :
94
- es_cluster :
95
- inputs :
96
- - apache_sampler
97
- type : elasticsearch
98
- endpoints : ['http://79.12.221.222:9200']
99
- bulk :
100
- index : vector-%Y-%m-%d
101
- s3_archives :
102
- inputs :
103
- - apache_parser
104
- type : aws_s3
105
- region : us-east-1
106
- bucket : my-log-archives
107
- key_prefix : date=%Y-%m-%d
108
- compression : gzip
109
- framing :
110
- method : newline_delimited
111
- encoding :
112
- codec : json
113
- batch :
114
- max_bytes : 10000000
115
- ` ` `
116
-
117
134
{{< /tab >}}
118
135
{{< tab title="vector.json" >}}
119
136
@@ -285,89 +302,98 @@ method. For most Linux-based systems, the file can be found at
285
302
You can pass multiple configuration files when starting Vector:
286
303
287
304
``` shell
288
- vector --config vector1.toml --config vector2.toml
305
+ vector --config vector1.yaml --config vector2.yaml
289
306
```
290
307
291
308
Or using a [ globbing syntax] [ glob ] :
292
309
293
310
``` shell
294
- vector --config /etc/vector/* .toml
311
+ vector --config /etc/vector/* .yaml
295
312
```
296
313
297
314
#### Automatic namespacing
298
315
299
316
You can also split your configuration by grouping the components by their type, one directory per component type, where the file name is used as the component id. For example:
300
317
301
- {{< tabs default="vector.toml " >}}
302
- {{< tab title="vector.toml " >}}
318
+ {{< tabs default="vector.yaml " >}}
319
+ {{< tab title="vector.yaml " >}}
303
320
304
- ``` toml
321
+ ``` yaml
305
322
# Set global options
306
- data_dir = " /var/lib/vector"
323
+ data_dir : " /var/lib/vector"
307
324
308
325
# Vector's API (disabled by default)
309
326
# Enable and try it out with the `vector top` command
310
- [ api ]
311
- enabled = false
312
- # address = "127.0.0.1:8686"
327
+ api :
328
+ enabled : false
329
+ # address: "127.0.0.1:8686"
313
330
```
314
331
315
332
{{< /tab >}}
316
- {{< tab title="sources/apache_logs.toml " >}}
333
+ {{< tab title="sources/apache_logs.yaml " >}}
317
334
318
- ``` toml
335
+ ``` yaml
319
336
# Ingest data by tailing one or more files
320
- type = " file"
321
- include = [" /var/log/apache2/*.log" ] # supports globbing
322
- ignore_older = 86400 # 1 day
337
+ type : " file"
338
+ include : ["/var/log/apache2/*.log"] # supports globbing
339
+ ignore_older : 86400 # 1 day
323
340
` ` `
324
341
325
342
{{< /tab >}}
326
- {{< tab title="transforms/apache_parser.toml " >}}
343
+ {{< tab title="transforms/apache_parser.yaml " >}}
327
344
328
- ``` toml
345
+ ` ` ` yaml
329
346
# Structure and parse via Vector Remap Language
330
- inputs = [" apache_logs" ]
331
- type = " remap"
332
- source = '''
333
- . = parse_apache_log(.message)
347
+ inputs :
348
+ - " apache_logs"
349
+ type : " remap"
350
+ source : |
351
+ . = parse_apache_log(.message)
334
352
` ` `
335
353
336
354
{{< /tab >}}
337
- {{< tab title="transforms/apache_sampler.toml " >}}
355
+ {{< tab title="transforms/apache_sampler.yaml " >}}
338
356
339
- ```toml
357
+ ` ` ` yaml
340
358
# Sample the data to save on cost
341
- inputs = ["apache_parser"]
342
- type = "sample"
343
- rate = 2 # only keep 50% (1/`rate`)
359
+ inputs :
360
+ - " apache_parser"
361
+ type : " sample"
362
+ rate : 2 # only keep 50% (1/`rate`)
344
363
` ` `
345
364
346
365
{{< /tab >}}
347
- {{< tab title="sinks/es_cluster.toml " >}}
366
+ {{< tab title="sinks/es_cluster.yaml " >}}
348
367
349
- ```toml
368
+ ` ` ` yaml
350
369
# Send structured data to a short-term storage
351
- inputs = ["apache_sampler"] # only take sampled data
352
- type = "elasticsearch"
353
- endpoints = ["http://79.12.221.222:9200"] # local or external host
354
- bulk.index = "vector-%Y-%m-%d" # daily indices
370
+ inputs :
371
+ - " apache_sampler" # only take sampled data
372
+ type : " elasticsearch"
373
+ endpoints :
374
+ - " http://79.12.221.222:9200" # local or external host
375
+ bulk :
376
+ index : " vector-%Y-%m-%d" # daily indices
355
377
` ` `
356
378
357
379
{{< /tab >}}
358
- {{< tab title="sinks/s3_archives.toml " >}}
380
+ {{< tab title="sinks/s3_archives.yaml " >}}
359
381
360
- ```toml
382
+ ` ` ` yaml
361
383
# Send structured data to a cost-effective long-term storage
362
- inputs = ["apache_parser"] # don't sample for S3
363
- type = "aws_s3"
364
- region = "us-east-1"
365
- bucket = "my-log-archives"
366
- key_prefix = "date=%Y-%m-%d" # daily partitions, hive friendly format
367
- compression = "gzip" # compress final objects
368
- framing.method = "newline_delimited" # new line delimited...
369
- encoding.codec = "json" # ...JSON
370
- batch.max_bytes = 10000000 # 10mb uncompressed
384
+ inputs :
385
+ - " apache_parser" # don't sample for S3
386
+ type : " aws_s3"
387
+ region : " us-east-1"
388
+ bucket : " my-log-archives"
389
+ key_prefix : " date=%Y-%m-%d" # daily partitions, hive-friendly format
390
+ compression : " gzip" # compress final objects
391
+ framing :
392
+ method : " newline_delimited" # new line delimited...
393
+ encoding :
394
+ codec : " json" # ...JSON
395
+ batch :
396
+ max_bytes : 10000000 # 10mb uncompressed
371
397
` ` `
372
398
373
399
{{< /tab >}}
@@ -384,26 +410,28 @@ vector --config-dir /etc/vector
384
410
Vector supports wildcards (`*`) in component IDs when building your topology.
385
411
For example :
386
412
387
- ```toml
388
- [sources.app1_logs]
389
- type = "file"
390
- includes = ["/var/log/app1.log"]
413
+ ` ` ` yaml
414
+ sources:
415
+ app1_logs:
416
+ type: "file"
417
+ includes: ["/var/log/app1.log"]
391
418
392
- [sources. app2_logs]
393
- type = "file"
394
- includes = ["/var/log/app.log"]
419
+ app2_logs:
420
+ type: "file"
421
+ includes: ["/var/log/app.log"]
395
422
396
- [sources. system_logs]
397
- type = "file"
398
- includes = ["/var/log/system.log"]
423
+ system_logs:
424
+ type: "file"
425
+ includes: ["/var/log/system.log"]
399
426
400
- [sinks.app_logs]
401
- type = "datadog_logs"
402
- inputs = ["app*"]
427
+ sinks:
428
+ app_logs:
429
+ type: "datadog_logs"
430
+ inputs: ["app*"]
403
431
404
- [sinks. archive]
405
- type = "aws_s3"
406
- inputs = ["app*", "system_logs"]
432
+ archive:
433
+ type: "aws_s3"
434
+ inputs: ["app*", "system_logs"]
407
435
` ` `
408
436
409
437
# # Sections
0 commit comments