Skip to content

chore(config): Update field labels for commonly used sources and transforms #17517

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/vector-config-common/src/human_friendly.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ static WELL_KNOWN_ACRONYMS: Lazy<HashSet<String>> = Lazy::new(|| {
"api", "amqp", "aws", "ec2", "ecs", "gcp", "hec", "http", "https", "nats", "nginx", "s3",
"sqs", "tls", "ssl", "otel", "gelf", "csv", "json", "rfc3339", "lz4", "us", "eu", "bsd",
"vrl", "tcp", "udp", "id", "uuid", "kms", "uri", "url", "acp", "uid", "ip", "pid",
"ndjson", "ewma", "rtt", "cpu", "acl", "imds", "acl", "alpn",
"ndjson", "ewma", "rtt", "cpu", "acl", "imds", "acl", "alpn", "sasl",
];

acronyms.iter().map(|s| s.to_lowercase()).collect()
Expand Down
1 change: 1 addition & 0 deletions lib/vector-core/src/tcp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use vector_config::configurable_component;
#[configurable_component]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
#[configurable(metadata(docs::human_name = "Wait Time"))]
pub struct TcpKeepaliveConfig {
/// The time to wait before starting to send TCP keepalive probes on an idle connection.
#[configurable(metadata(docs::type_unit = "seconds"))]
Expand Down
1 change: 1 addition & 0 deletions src/sources/apache_metrics/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ pub struct ApacheMetricsConfig {
/// The interval between scrapes.
#[serde(default = "default_scrape_interval_secs")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
scrape_interval_secs: Duration,

/// The namespace of the metric.
Expand Down
1 change: 1 addition & 0 deletions src/sources/aws_ecs_metrics/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ pub struct AwsEcsMetricsSourceConfig {
/// The interval between scrapes, in seconds.
#[serde(default = "default_scrape_interval_secs")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
scrape_interval_secs: Duration,

/// The namespace of the metric.
Expand Down
4 changes: 2 additions & 2 deletions src/sources/aws_kinesis_firehose/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,15 @@ pub struct AwsKinesisFirehoseConfig {
#[configurable(metadata(docs::examples = "localhost:443"))]
address: SocketAddr,

/// An optional access key to authenticate requests against.
/// An access key to authenticate requests against.
///
/// AWS Kinesis Firehose can be configured to pass along a user-configurable access key with each request. If
/// configured, `access_key` should be set to the same value. Otherwise, all requests are allowed.
#[configurable(deprecated = "This option has been deprecated, use `access_keys` instead.")]
#[configurable(metadata(docs::examples = "A94A8FE5CCB19BA61C4C08"))]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know if it's in scope for this PR, but I would support putting a docs::hidden metadata here to hide this in the config builder. We shouldn't be showing deprecated options like this to users.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would also drop it from the Vector docs - which might not be desired until we actually remove it. That said, it's probably been deprecated for more than a release and can be removed now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a list of fields that are noted as deprecated. This is one of them, so I'm thinking we can address them all in one PR?

access_key: Option<SensitiveString>,

/// An optional list of access keys to authenticate requests against.
/// A list of access keys to authenticate requests against.
///
/// AWS Kinesis Firehose can be configured to pass along a user-configurable access key with each request. If
/// configured, `access_keys` should be set to the same value. Otherwise, all requests are allowed.
Expand Down
1 change: 1 addition & 0 deletions src/sources/aws_s3/sqs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ pub(super) struct Config {
#[serde(default = "default_visibility_timeout_secs")]
#[derivative(Default(value = "default_visibility_timeout_secs()"))]
#[configurable(metadata(docs::type_unit = "seconds"))]
#[configurable(metadata(docs::human_name = "Visibility Timeout"))]
pub(super) visibility_timeout_secs: u32,

/// Whether to delete the message once it is processed.
Expand Down
1 change: 1 addition & 0 deletions src/sources/aws_sqs/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ pub struct AwsSqsConfig {
#[serde(default = "default_visibility_timeout_secs")]
#[derivative(Default(value = "default_visibility_timeout_secs()"))]
#[configurable(metadata(docs::type_unit = "seconds"))]
#[configurable(metadata(docs::human_name = "Visibility Timeout"))]
pub(super) visibility_timeout_secs: u32,

/// Whether to delete the message once it is processed.
Expand Down
4 changes: 4 additions & 0 deletions src/sources/gcp_pubsub.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,13 +163,15 @@ pub struct PubsubConfig {
/// are all busy and so open a new stream.
#[serde(default = "default_poll_time")]
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Poll Time"))]
pub poll_time_seconds: Duration,

/// The acknowledgement deadline, in seconds, to use for this stream.
///
/// Messages that are not acknowledged when this deadline expires may be retransmitted.
#[serde(default = "default_ack_deadline")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[configurable(metadata(docs::human_name = "Acknowledgement Deadline"))]
pub ack_deadline_secs: Duration,

/// The acknowledgement deadline, in seconds, to use for this stream.
Expand All @@ -183,6 +185,7 @@ pub struct PubsubConfig {
/// The amount of time, in seconds, to wait between retry attempts after an error.
#[serde(default = "default_retry_delay")]
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Retry Delay"))]
pub retry_delay_secs: Duration,

/// The amount of time, in seconds, to wait between retry attempts after an error.
Expand All @@ -196,6 +199,7 @@ pub struct PubsubConfig {
/// `60`, you may see periodic errors sent from the server.
#[serde(default = "default_keepalive")]
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Keepalive"))]
pub keepalive_secs: Duration,

/// The namespace to use for logs. This overrides the global setting.
Expand Down
1 change: 1 addition & 0 deletions src/sources/http_client/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ pub struct HttpClientConfig {
#[serde(default = "default_interval")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[serde(rename = "scrape_interval_secs")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
pub interval: Duration,

/// Custom parameters for the HTTP request query string.
Expand Down
4 changes: 4 additions & 0 deletions src/sources/kafka.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,26 +107,30 @@ pub struct KafkaSourceConfig {
#[configurable(metadata(docs::examples = 5000, docs::examples = 10000))]
#[configurable(metadata(docs::advanced))]
#[serde(default = "default_session_timeout_ms")]
#[configurable(metadata(docs::human_name = "Session Timeout"))]
session_timeout_ms: Duration,

/// Timeout for network requests.
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[configurable(metadata(docs::examples = 30000, docs::examples = 60000))]
#[configurable(metadata(docs::advanced))]
#[serde(default = "default_socket_timeout_ms")]
#[configurable(metadata(docs::human_name = "Socket Timeout"))]
socket_timeout_ms: Duration,

/// Maximum time the broker may wait to fill the response.
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[configurable(metadata(docs::examples = 50, docs::examples = 100))]
#[configurable(metadata(docs::advanced))]
#[serde(default = "default_fetch_wait_max_ms")]
#[configurable(metadata(docs::human_name = "Max Fetch Wait Time"))]
fetch_wait_max_ms: Duration,

/// The frequency that the consumer offsets are committed (written) to offset storage.
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[serde(default = "default_commit_interval_ms")]
#[configurable(metadata(docs::examples = 5000, docs::examples = 10000))]
#[configurable(metadata(docs::human_name = "Commit Interval"))]
commit_interval_ms: Duration,

/// Overrides the name of the log field used to add the message key to each event.
Expand Down
1 change: 1 addition & 0 deletions src/sources/prometheus/scrape.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ pub struct PrometheusScrapeConfig {
#[serde(default = "default_interval")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[serde(rename = "scrape_interval_secs")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
interval: Duration,

/// The tag name added to each event representing the scraped instance's `host:port`.
Expand Down
10 changes: 7 additions & 3 deletions src/sources/splunk_hec/acknowledgements.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,28 +27,32 @@ pub struct HecAcknowledgementsConfig {
/// Enables end-to-end acknowledgements.
pub enabled: Option<bool>,

/// The maximum number of ack statuses pending query across all channels.
/// The maximum number of acknowledgement statuses pending query across all channels.
///
/// Equivalent to the `max_number_of_acked_requests_pending_query` Splunk HEC setting.
///
/// Minimum of `1`.
#[configurable(metadata(docs::human_name = "Max Number of Pending Acknowledgements"))]
pub max_pending_acks: NonZeroU64,

/// The maximum number of Splunk HEC channels clients can use with this source.
///
/// Minimum of `1`.
#[configurable(metadata(docs::human_name = "Max Number of Acknowledgement Channels"))]
pub max_number_of_ack_channels: NonZeroU64,

/// The maximum number of ack statuses pending query for a single channel.
/// The maximum number of acknowledgement statuses pending query for a single channel.
///
/// Equivalent to the `max_number_of_acked_requests_pending_query_per_ack_channel` Splunk HEC setting.
///
/// Minimum of `1`.
#[configurable(metadata(docs::human_name = "Max Number of Pending Acknowledgements Per Channel"))]
pub max_pending_acks_per_channel: NonZeroU64,

/// Whether or not to remove channels after idling for `max_idle_time` seconds.
///
/// A channel is idling if it is not used for sending data or querying ack statuses.
/// A channel is idling if it is not used for sending data or querying acknowledgement statuses.
#[configurable(metadata(docs::human_name = "Acknowledgement Idle Cleanup"))]
pub ack_idle_cleanup: bool,

/// The amount of time, in seconds, a channel is allowed to idle before removal.
Expand Down
1 change: 1 addition & 0 deletions src/sources/util/multiline_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ pub struct MultilineConfig {
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[configurable(metadata(docs::examples = 1000))]
#[configurable(metadata(docs::examples = 600000))]
#[configurable(metadata(docs::human_name = "Timeout"))]
pub timeout_ms: Duration,
}

Expand Down
1 change: 1 addition & 0 deletions src/transforms/aggregate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ pub struct AggregateConfig {
///
/// During this time frame, metrics with the same series data (name, namespace, tags, and so on) are aggregated.
#[serde(default = "default_interval_ms")]
#[configurable(metadata(docs::human_name = "Flush Interval"))]
pub interval_ms: u64,
}

Expand Down
2 changes: 2 additions & 0 deletions src/transforms/reduce/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,14 @@ pub struct ReduceConfig {
#[serde(default = "default_expire_after_ms")]
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[derivative(Default(value = "default_expire_after_ms()"))]
#[configurable(metadata(docs::human_name = "Expire After"))]
pub expire_after_ms: Duration,

/// The interval to check for and flush any expired events, in milliseconds.
#[serde(default = "default_flush_period_ms")]
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[derivative(Default(value = "default_flush_period_ms()"))]
#[configurable(metadata(docs::human_name = "Flush Period"))]
pub flush_period_ms: Duration,

/// The maximum number of events to group together.
Expand Down
3 changes: 3 additions & 0 deletions src/transforms/remap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ pub struct RemapConfig {
/// Additionally, dropped events can potentially be diverted to a specially named output for
/// further logging and analysis by setting `reroute_dropped`.
#[serde(default = "crate::serde::default_false")]
#[configurable(metadata(docs::human_name = "Drop Event on Error"))]
pub drop_on_error: bool,

/// Drops any event that is manually aborted during processing.
Expand All @@ -117,6 +118,7 @@ pub struct RemapConfig {
///
/// [vrl_docs_abort]: https://vector.dev/docs/reference/vrl/expressions/#abort
#[serde(default = "crate::serde::default_true")]
#[configurable(metadata(docs::human_name = "Drop Event on Abort"))]
pub drop_on_abort: bool,

/// Reroutes dropped events to a named output instead of halting processing on them.
Expand All @@ -129,6 +131,7 @@ pub struct RemapConfig {
/// to a specially-named output, `dropped`. The original event is annotated with additional
/// fields describing why the event was dropped.
#[serde(default = "crate::serde::default_false")]
#[configurable(metadata(docs::human_name = "Reroute Dropped Events"))]
pub reroute_dropped: bool,

#[configurable(derived, metadata(docs::hidden))]
Expand Down
1 change: 1 addition & 0 deletions src/transforms/tag_cardinality_limit/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ pub struct BloomFilterConfig {
/// The larger the cache size, the less likely it is to have a false positive, or a case where
/// we allow a new value for tag even after we have reached the configured limits.
#[serde(default = "default_cache_size")]
#[configurable(metadata(docs::human_name = "Cache Size per Key"))]
pub cache_size_per_key: usize,
}

Expand Down
1 change: 1 addition & 0 deletions src/transforms/throttle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ pub struct ThrottleConfig {

/// The time window in which the configured `threshold` is applied, in seconds.
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Time Window"))]
window_secs: Duration,

/// The value to group events into separate buckets to be rate limited independently.
Expand Down