From c1afa3651018bf0e7c45bdb01284d557ac146997 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 20 Aug 2024 13:51:39 +0200 Subject: [PATCH 1/6] Update converter README --- comp/otelcol/converter/README.md | 22 ++++++++++++++++ .../converter/examples/extensions.yaml | 25 +++++++++++++++++++ .../converter/examples/infraattributes.yaml | 25 +++++++++++++++++++ .../converter/examples/prometheus.yaml | 19 ++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 comp/otelcol/converter/examples/extensions.yaml create mode 100644 comp/otelcol/converter/examples/infraattributes.yaml create mode 100644 comp/otelcol/converter/examples/prometheus.yaml diff --git a/comp/otelcol/converter/README.md b/comp/otelcol/converter/README.md index 8c33b64962ede0..1c7811d5d57986 100644 --- a/comp/otelcol/converter/README.md +++ b/comp/otelcol/converter/README.md @@ -29,3 +29,25 @@ For any prometheus receiver collecting collector health metrics, and sending the `GetProvidedConf` and `GetEnhancedConf` return the string representation of the user provided and autoconfigured conf respectively. Currently, these APIs have two limitations: - They do not redact sensitive data - They do not provide the effective config (including defaults...etc) + +## Opting out of converter + +It is possible to opt out of the converter by setting env var `DD_OTELCOLLECTOR_CONVERTER_ENABLED` or agent config `otelcollector.converter.enabled` to `false` (`true` by default). + +You are still able to add the components added by the converter manually: + +### Extensions + +Please refer to the following example in order to manually set the `pprof`, `health_check`, `zpages` and `datadog` extensions: [extensions.yaml](examples/extensions.yaml). Please refer to the extensions README.md for additional information about the components: +- [pprof](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/pprofextension/README.md) +- [health_check](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/healthcheckextension/README.md) +- [zpages](https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/zpagesextension/README.md) +- [datadog](../extension/README.md) + +### Prometheus Receiver + +Please refer to the following example in order to manually set the prometheus receiver: [prometheus.yaml](examples/prometheus.yaml). Please refer to the receivers [README.md](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) for additional information about the component. + +### Infra Attributes Processor + +Please refer to the following example in order to manually set the infraattributes processor: [infraattributes.yaml](examples/infraattributes.yaml). Please refer to the processors [README.md](../otlp/components/processor/infraattributesprocessor/README.md) for additional information about the component. diff --git a/comp/otelcol/converter/examples/extensions.yaml b/comp/otelcol/converter/examples/extensions.yaml new file mode 100644 index 00000000000000..f2758cd57c06e1 --- /dev/null +++ b/comp/otelcol/converter/examples/extensions.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + +exporters: + datadog: + +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +service: + extensions: [pprof/user-defined, zpages/user-defined, health_check/user-defined, datadog/user-defined] + pipelines: + traces: + receivers: [otlp] + exporters: [datadog] + metrics: + receivers: [otlp] + exporters: [datadog] + logs: + receivers: [otlp] + exporters: [datadog] \ No newline at end of file diff --git a/comp/otelcol/converter/examples/infraattributes.yaml b/comp/otelcol/converter/examples/infraattributes.yaml new file mode 100644 index 00000000000000..44347e2edf87fa --- /dev/null +++ b/comp/otelcol/converter/examples/infraattributes.yaml @@ -0,0 +1,25 @@ +receivers: + otlp: + +exporters: + datadog: + api: + key: 12345 + +processors: + infraattributes/user-defined: + +service: + pipelines: + traces: + receivers: [otlp] + processors: [infraattributes/user-defined] + exporters: [datadog] + metrics: + receivers: [otlp] + processors: [infraattributes/user-defined] + exporters: [datadog] + logs: + receivers: [otlp] + processors: [infraattributes/user-defined] + exporters: [datadog] diff --git a/comp/otelcol/converter/examples/prometheus.yaml b/comp/otelcol/converter/examples/prometheus.yaml new file mode 100644 index 00000000000000..43903d24dec949 --- /dev/null +++ b/comp/otelcol/converter/examples/prometheus.yaml @@ -0,0 +1,19 @@ +receivers: + prometheus/user-defined: + config: + scrape_configs: + - job_name: 'datadog-agent' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:8888'] + +exporters: + datadog: + api: + key: 12345 + +service: + pipelines: + metrics: + receivers: [nop, prometheus/user-defined] + exporters: [datadog] From 041a81d0617559b5948ea287bac347d80b0ab47c Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 20 Aug 2024 14:49:42 +0200 Subject: [PATCH 2/6] add description --- comp/otelcol/converter/README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/comp/otelcol/converter/README.md b/comp/otelcol/converter/README.md index 1c7811d5d57986..e770121ac2bef3 100644 --- a/comp/otelcol/converter/README.md +++ b/comp/otelcol/converter/README.md @@ -32,22 +32,20 @@ For any prometheus receiver collecting collector health metrics, and sending the ## Opting out of converter -It is possible to opt out of the converter by setting env var `DD_OTELCOLLECTOR_CONVERTER_ENABLED` or agent config `otelcollector.converter.enabled` to `false` (`true` by default). - -You are still able to add the components added by the converter manually: +It is possible to opt out of the converter by setting env var `DD_OTELCOLLECTOR_CONVERTER_ENABLED` or agent config `otelcollector.converter.enabled` to `false` (`true` by default). Please note that by doing so, you are removing functionality including flare collection from otel-agent, health metrics from collector, or infra level tagging on your telemetry data. If you want to opt out of some components, you can disable all and add the components that you require manually: ### Extensions Please refer to the following example in order to manually set the `pprof`, `health_check`, `zpages` and `datadog` extensions: [extensions.yaml](examples/extensions.yaml). Please refer to the extensions README.md for additional information about the components: -- [pprof](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/pprofextension/README.md) -- [health_check](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/healthcheckextension/README.md) -- [zpages](https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/zpagesextension/README.md) -- [datadog](../extension/README.md) +- [pprof](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/extension/pprofextension/README.md): Enables collecting collector profiles at a defined endpoint. +- [health_check](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/healthcheckextension/README.md): Enables an HTTP url that can be probed to check the status of the OpenTelemetry Collector. +- [zpages](https://github.com/open-telemetry/opentelemetry-collector/blob/main/extension/zpagesextension/README.md): Enables an extension that serves zPages, an HTTP endpoint that provides live data for debugging different components +- [datadog](../extension/README.md): Enables otel-agent information to be collected in the datadog-agent flare. ### Prometheus Receiver -Please refer to the following example in order to manually set the prometheus receiver: [prometheus.yaml](examples/prometheus.yaml). Please refer to the receivers [README.md](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) for additional information about the component. +The Prometheus receiver scrapes prometheus endpoints. This is used to collect the collectors internal health metrics, by adding a job that scrapes the service telemetry metrics endpoint (configurable via `service::telemetry::metrics`). Please refer to the following example in order to manually set the prometheus receiver: [prometheus.yaml](examples/prometheus.yaml). Please refer to the receivers [README.md](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) for additional information about the component. ### Infra Attributes Processor -Please refer to the following example in order to manually set the infraattributes processor: [infraattributes.yaml](examples/infraattributes.yaml). Please refer to the processors [README.md](../otlp/components/processor/infraattributesprocessor/README.md) for additional information about the component. +The infraattributes processor is used to add infra level tags collected by the datadog-agent to your telemetry data. Please refer to the following example in order to manually set the infraattributes processor: [infraattributes.yaml](examples/infraattributes.yaml). Please refer to the processors [README.md](../otlp/components/processor/infraattributesprocessor/README.md) for additional information about the component. From 37aa27b7b5111ff3f116883b860ca790bc45a4d7 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 20 Aug 2024 14:52:50 +0200 Subject: [PATCH 3/6] fix --- comp/otelcol/converter/examples/extensions.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/comp/otelcol/converter/examples/extensions.yaml b/comp/otelcol/converter/examples/extensions.yaml index f2758cd57c06e1..60d0063f70223f 100644 --- a/comp/otelcol/converter/examples/extensions.yaml +++ b/comp/otelcol/converter/examples/extensions.yaml @@ -2,7 +2,9 @@ receivers: otlp: exporters: - datadog: + datadog: + api: + key: 12345 extensions: pprof/user-defined: From 79b995c8d809d62784b7351c00f489bdc566fd57 Mon Sep 17 00:00:00 2001 From: Mackenzie <63265430+mackjmr@users.noreply.github.com> Date: Tue, 20 Aug 2024 16:30:44 +0200 Subject: [PATCH 4/6] Update comp/otelcol/converter/examples/prometheus.yaml Co-authored-by: Yang Song --- comp/otelcol/converter/examples/prometheus.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comp/otelcol/converter/examples/prometheus.yaml b/comp/otelcol/converter/examples/prometheus.yaml index 43903d24dec949..1e69237b20a0e3 100644 --- a/comp/otelcol/converter/examples/prometheus.yaml +++ b/comp/otelcol/converter/examples/prometheus.yaml @@ -15,5 +15,5 @@ exporters: service: pipelines: metrics: - receivers: [nop, prometheus/user-defined] + receivers: [prometheus/user-defined] exporters: [datadog] From c9a5565d23b05dca856fddf627149d17f0bf61bb Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 20 Aug 2024 16:33:23 +0200 Subject: [PATCH 5/6] add datadogconnector --- comp/otelcol/converter/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/comp/otelcol/converter/README.md b/comp/otelcol/converter/README.md index e770121ac2bef3..e54e339bef1b13 100644 --- a/comp/otelcol/converter/README.md +++ b/comp/otelcol/converter/README.md @@ -24,6 +24,10 @@ If it finds datadogexporters which are not defined in a pipeline with the promet For any prometheus receiver collecting collector health metrics, and sending these to Datadog, it will update the job name to `datadog-agent`. This ensures the health metrics are tagged by `service:datadog-agent` and differentiable from collector health metrics. +### Datadog Connector + +The converter will automatically set `datadogconnector` config `trace.span_name_as_resource_name` to true in any datadog connectors in your configuration. + ## Provided and enhanced config `GetProvidedConf` and `GetEnhancedConf` return the string representation of the user provided and autoconfigured conf respectively. Currently, these APIs have two limitations: From 0637a7fea17dc76fee3bd37845d0e97bef916cf1 Mon Sep 17 00:00:00 2001 From: mackjmr Date: Tue, 27 Aug 2024 13:35:04 +0200 Subject: [PATCH 6/6] add api_key instructions --- comp/otelcol/converter/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/comp/otelcol/converter/README.md b/comp/otelcol/converter/README.md index e54e339bef1b13..31c0c53f7176b5 100644 --- a/comp/otelcol/converter/README.md +++ b/comp/otelcol/converter/README.md @@ -24,6 +24,10 @@ If it finds datadogexporters which are not defined in a pipeline with the promet For any prometheus receiver collecting collector health metrics, and sending these to Datadog, it will update the job name to `datadog-agent`. This ensures the health metrics are tagged by `service:datadog-agent` and differentiable from collector health metrics. +### API Key and API Site + +If `api_key` is unset, set to an empty string or set to a secret, the converter will fetch the api key from the agent configuration. It will also fetch the the site from the agent config if unset in collector. + ### Datadog Connector The converter will automatically set `datadogconnector` config `trace.span_name_as_resource_name` to true in any datadog connectors in your configuration.