[exporter/datadog] Upgrade exporter.datadogexporter.UseLogsAgentExporter feature flag to beta (open-telemetry#34420)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
Use Datadog Agent logs pipeline by default for exporting logs to
Datadog. Upgrades `exporter.datadogexporter.UseLogsAgentExporter`
feature flag to beta.

Note: if users have `logs::dump_payloads` enabled in their config after
upgrading, an error will be raised because this config option cannot be
used with the logs agent exporter:
https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/f1aadc6c9c5d117eb61898cd2f1969a279a7f80e/exporter/datadogexporter/config.go#L658

**Link to tracking Issue:** <Issue number if applicable>
Fixes
open-telemetry#34366
and introduces general performance improvements.

**Testing:** <Describe what testing was performed and which tests were
added.>

**Documentation:** <Describe the documentation added.>

---------

Co-authored-by: Pablo Baeyens <[email protected]>

Author: liustanley

.chloggen/stanley.liu_logs-agent-beta-breaking.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/datadog
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: The `logs::dump_payloads` config option is invalid when the Datadog Agent logs pipeline is enabled (now enabled by default).
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34420]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: An error will be raised if `logs::dump_payloads` is set while the Datadog Agent logs pipeline is enabled. To avoid this error, remove the `logs::dump_payloads` config option or temporarily disable the `exporter.datadogexporter.UseLogsAgentExporter` feature gate.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

.chloggen/stanley.liu_logs-agent-beta.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: exporter/datadog
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use Datadog Agent logs pipeline by default for exporting logs to Datadog. Upgrades `exporter.datadogexporter.UseLogsAgentExporter` feature flag to beta.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34420]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/datadogexporter/config.go

@@ -323,21 +323,21 @@ type LogsConfig struct {
 	confignet.TCPAddrConfig `mapstructure:",squash"`
 
 	// DumpPayloads report whether payloads should be dumped when logging level is debug.
-	// Note: this config option does not apply when enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+	// Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is enabled (now enabled by default).
 	// Deprecated: This config option is not supported in the Datadog Agent logs pipeline.
 	DumpPayloads bool `mapstructure:"dump_payloads"`
 
 	// UseCompression enables the logs agent to compress logs before sending them.
-	// Note: this config option does not apply unless enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+	// Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is disabled.
 	UseCompression bool `mapstructure:"use_compression"`
 
 	// CompressionLevel accepts values from 0 (no compression) to 9 (maximum compression but higher resource usage).
 	// Only takes effect if UseCompression is set to true.
-	// Note: this config option does not apply unless enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+	// Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is disabled.
 	CompressionLevel int `mapstructure:"compression_level"`
 
 	// BatchWait represents the maximum time the logs agent waits to fill each batch of logs before sending.
-	// Note: this config option does not apply unless enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+	// Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is disabled.
 	BatchWait int `mapstructure:"batch_wait"`
 }
 

exporter/datadogexporter/examples/collector.yaml

@@ -457,28 +457,28 @@ exporters:
       ## If set to true, payloads will be dumped when logging level is set to debug. Please note that
       ## This may result in an escaping loop if a filelog receiver is watching the collector log output.
       ## See: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16380
-      ## Note: this config option does not apply when enabling `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+      ## Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is enabled (now enabled by default).
       #
       # dump_payloads: false
 
       ## @param use_compression - boolean - optional - default: true
       ## This parameter is available when sending logs with HTTPS. If enabled, the logs agent
       ## compresses logs before sending them.
-      ## Note: this config option does not apply unless enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+      ## Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is disabled.
       #
       # use_compression: true
 
       ## @param compression_level - integer - optional - default: 6
       ## The compression_level parameter accepts values from 0 (no compression)
       ## to 9 (maximum compression but higher resource usage). Only takes effect if
       ## `use_compression` is set to `true`.
-      ## Note: this config option does not apply unless enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+      ## Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is disabled.
       #
       # compression_level: 6
 
       ## @param batch_wait - integer - optional - default: 5
       ## The maximum time the logs agent waits to fill each batch of logs before sending.
-      ## Note: this config option does not apply unless enabling the `exporter.datadogexporter.UseLogsAgentExporter` feature flag.
+      ## Note: this config option does not apply when the `exporter.datadogexporter.UseLogsAgentExporter` feature flag is disabled.
       #
       # batch_wait: 5
 

exporter/datadogexporter/factory.go

@@ -42,7 +42,7 @@ import (
 
 var logsAgentExporterFeatureGate = featuregate.GlobalRegistry().MustRegister(
 	"exporter.datadogexporter.UseLogsAgentExporter",
-	featuregate.StageAlpha,
+	featuregate.StageBeta,
 	featuregate.WithRegisterDescription("When enabled, datadogexporter uses the Datadog agent logs pipeline for exporting logs."),
 	featuregate.WithRegisterFromVersion("v0.100.0"),
 )

exporter/datadogexporter/factory_test.go

@@ -123,67 +123,6 @@ func TestCreateDefaultConfig(t *testing.T) {
 	assert.NoError(t, componenttest.CheckConfigStruct(cfg))
 }
 
-// Test that the factory creates the default configuration
-func TestCreateDefaultConfigLogsAgent(t *testing.T) {
-	err := featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", true)
-	assert.NoError(t, err)
-	factory := NewFactory()
-	cfg := factory.CreateDefaultConfig()
-
-	assert.Equal(t, &Config{
-		ClientConfig:  defaultClientConfig(),
-		BackOffConfig: configretry.NewDefaultBackOffConfig(),
-		QueueSettings: exporterhelper.NewDefaultQueueSettings(),
-
-		API: APIConfig{
-			Site: "datadoghq.com",
-		},
-
-		Metrics: MetricsConfig{
-			TCPAddrConfig: confignet.TCPAddrConfig{
-				Endpoint: "https://api.datadoghq.com",
-			},
-			DeltaTTL: 3600,
-			HistConfig: HistogramConfig{
-				Mode:             "distributions",
-				SendAggregations: false,
-			},
-			SumConfig: SumConfig{
-				CumulativeMonotonicMode:        CumulativeMonotonicSumModeToDelta,
-				InitialCumulativeMonotonicMode: InitialValueModeAuto,
-			},
-			SummaryConfig: SummaryConfig{
-				Mode: SummaryModeGauges,
-			},
-		},
-
-		Traces: TracesConfig{
-			TCPAddrConfig: confignet.TCPAddrConfig{
-				Endpoint: "https://trace.agent.datadoghq.com",
-			},
-			IgnoreResources: []string{},
-		},
-		Logs: LogsConfig{
-			TCPAddrConfig: confignet.TCPAddrConfig{
-				Endpoint: "https://http-intake.logs.datadoghq.com",
-			},
-			UseCompression:   true,
-			CompressionLevel: 6,
-			BatchWait:        5,
-		},
-
-		HostMetadata: HostMetadataConfig{
-			Enabled:        true,
-			HostnameSource: HostnameSourceConfigOrSystem,
-		},
-		OnlyMetadata: false,
-	}, cfg, "failed to create default config")
-
-	assert.NoError(t, componenttest.CheckConfigStruct(cfg))
-	err = featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", false)
-	assert.NoError(t, err)
-}
-
 func TestLoadConfig(t *testing.T) {
 	t.Parallel()
 
@@ -500,6 +439,8 @@ func TestCreateAPIMetricsExporter(t *testing.T) {
 }
 
 func TestCreateAPIExporterFailOnInvalidKey_Zorkian(t *testing.T) {
+	featuregateErr := featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", false)
+	assert.NoError(t, featuregateErr)
 	server := testutil.DatadogServerMock(testutil.ValidateAPIKeyEndpointInvalid)
 	defer server.Close()
 
@@ -577,9 +518,13 @@ func TestCreateAPIExporterFailOnInvalidKey_Zorkian(t *testing.T) {
 		assert.NoError(t, err)
 		assert.NotNil(t, lexp)
 	})
+	featuregateErr = featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", true)
+	assert.NoError(t, featuregateErr)
 }
 
 func TestCreateAPIExporterFailOnInvalidKey(t *testing.T) {
+	featuregateErr := featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", false)
+	assert.NoError(t, featuregateErr)
 	server := testutil.DatadogServerMock(testutil.ValidateAPIKeyEndpointInvalid)
 	defer server.Close()
 
@@ -657,6 +602,8 @@ func TestCreateAPIExporterFailOnInvalidKey(t *testing.T) {
 		assert.NoError(t, err)
 		assert.NotNil(t, lexp)
 	})
+	featuregateErr = featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", true)
+	assert.NoError(t, featuregateErr)
 }
 
 func TestCreateAPILogsExporter(t *testing.T) {

exporter/datadogexporter/integrationtest/integration_test.go

@@ -466,10 +466,6 @@ func TestIntegrationLogs(t *testing.T) {
 	t.Setenv("SERVER_URL", server.URL)
 
 	// 2. Start in-process collector
-	assert.NoError(t, featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", true))
-	defer func() {
-		assert.NoError(t, featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", false))
-	}()
 	factories := getIntegrationTestComponents(t)
 	app := getIntegrationTestCollector(t, "integration_test_logs_config.yaml", factories)
 	go func() {

exporter/datadogexporter/logs_exporter_test.go

@@ -212,6 +212,8 @@ func TestLogsExporter(t *testing.T) {
 			},
 		},
 	}
+	featuregateErr := featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", false)
+	assert.NoError(t, featuregateErr)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			server := testutil.DatadogLogServerMock()
@@ -238,6 +240,8 @@ func TestLogsExporter(t *testing.T) {
 			assert.Equal(t, tt.want, server.LogsData)
 		})
 	}
+	featuregateErr = featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", true)
+	assert.NoError(t, featuregateErr)
 }
 
 func TestLogsAgentExporter(t *testing.T) {
@@ -497,8 +501,6 @@ func TestLogsAgentExporter(t *testing.T) {
 			},
 		},
 	}
-	err := featuregate.GlobalRegistry().Set("exporter.datadogexporter.UseLogsAgentExporter", true)
-	assert.NoError(t, err)
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			doneChannel := make(chan bool)