[exporter/elasticsearch] Add telemetry.log_request_body and telemetry.log_response_body config (open-telemetry#33854)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
- Add `telemetry.log_request_body` and `telemetry.log_response_body`
config for debugging. Debug log will contain field `request_body` and/or
`response_body` in the same log line instead of separate lines to avoid
interleaved log lines.
- Change "Request failed" log level to debug.

Output:
```
2024-07-02T14:09:24.983+0100	debug	elasticsearchexporter/elasticsearch_bulk.go:67	Request roundtrip completed.	{"kind": "exporter", "data_type": "logs", "name": "elasticsearch", "response_body": "{\"version\":{\"number\":\"1.2.3\"}}\n", "path": "/", "method": "GET", "duration": 0.000865486, "status": "200 OK"}
2024-07-02T14:09:24.984+0100	debug	elasticsearchexporter/elasticsearch_bulk.go:67	Request roundtrip completed.	{"kind": "exporter", "data_type": "logs", "name": "elasticsearch", "request_body": "{\"create\":{\"_index\":\"logs-test-idx\"}}\n{\"@timestamp\":\"2024-07-02T13:09:24.970187592Z\",\"Attributes\":{\"a\":\"test\",\"b\":5,\"batch_index\":\"batch_1\",\"c\":3,\"d\":true,\"item_index\":\"item_1\"},\"Body\":\"Load Generator Counter #0\",\"Scope\":{\"name\":\"\",\"version\":\"\"},\"SeverityNumber\":11,\"SeverityText\":\"INFO3\",\"TraceFlags\":1}\n{\"create\":{\"_index\":\"logs-test-idx\"}}\n{\"@timestamp\":\"2024-07-02T13:09:24.970187592Z\",\"Attributes\":{\"a\":\"test\",\"b\":5,\"batch_index\":\"batch_1\",\"c\":3,\"d\":true,\"item_index\":\"item_2\"},\"Body\":\"Load Generator Counter #1\",\"Scope\":{\"name\":\"\",\"version\":\"\"},\"SeverityNumber\":11,\"SeverityText\":\"INFO3\",\"TraceFlags\":1}\n", "response_body": "{\"took\":0,\"errors\":false,\"items\":[{\"create\":{\"_index\":\"logs-test-idx\",\"_id\":\"\",\"_version\":0,\"result\":\"\",\"status\":201,\"_seq_no\":0,\"_primary_term\":0,\"_shards\":{\"total\":0,\"successful\":0,\"failed\":0},\"error\":{\"type\":\"\",\"reason\":\"\",\"caused_by\":{\"type\":\"\",\"reason\":\"\"}}}},{\"create\":{\"_index\":\"logs-test-idx\",\"_id\":\"\",\"_version\":0,\"result\":\"\",\"status\":201,\"_seq_no\":0,\"_primary_term\":0,\"_shards\":{\"total\":0,\"successful\":0,\"failed\":0},\"error\":{\"type\":\"\",\"reason\":\"\",\"caused_by\":{\"type\":\"\",\"reason\":\"\"}}}}]}\n", "path": "/_bulk", "method": "POST", "duration": 0.000539979, "status": "200 OK"}
```

Required config to log
```
exporters:
  elasticsearch:
    telemetry:
      log_request_body: true
      log_response_body: true
    
service:
  telemetry:
    logs:
      level: debug
```

For easier analysis, limit the size of request body size. Use
`num_workers`=1 and lower `flush.bytes` and/or `flush.interval`.

**Link to tracking Issue:** <Issue number if applicable>

**Testing:** <Describe what testing was performed and which tests were
added.>

Manually verified with a modified integration test.

**Documentation:** <Describe the documentation added.>

Author: carsonip

.chloggen/elasticsearchexporter_telemetry_settings.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Introduce experimental `telemetry.log_request_body` and `telemetry.log_response_body` config
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33854]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/elasticsearchexporter/README.md

@@ -184,6 +184,16 @@ Settings related to node discovery are:
 
 Node discovery can be disabled by setting `discover.interval` to 0.
 
+### Telemetry settings
+
+The Elasticsearch Exporter's own telemetry settings for testing and debugging purposes.
+
+⚠️ This is experimental and may change at any time.
+
+- `telemetry`:
+  - `log_request_body` (default=false): Logs Elasticsearch client request body as a field in a log line at DEBUG level. It requires `service::telemetry::logs::level` to be set to `debug`. WARNING: Enabling this config may expose sensitive data.
+  - `log_response_body` (default=false): Logs Elasticsearch client response body as a field in a log line at DEBUG level. It requires `service::telemetry::logs::level` to be set to `debug`. WARNING: Enabling this config may expose sensitive data.
+
 ## Exporting metrics
 
 Metrics support is currently in development.

exporter/elasticsearchexporter/config.go

@@ -72,6 +72,15 @@ type Config struct {
 	Flush                   FlushSettings          `mapstructure:"flush"`
 	Mapping                 MappingsSettings       `mapstructure:"mapping"`
 	LogstashFormat          LogstashFormatSettings `mapstructure:"logstash_format"`
+
+	// TelemetrySettings contains settings useful for testing/debugging purposes
+	// This is experimental and may change at any time.
+	TelemetrySettings `mapstructure:"telemetry"`
+}
+
+type TelemetrySettings struct {
+	LogRequestBody  bool `mapstructure:"log_request_body"`
+	LogResponseBody bool `mapstructure:"log_response_body"`
 }
 
 type LogstashFormatSettings struct {

exporter/elasticsearchexporter/elasticsearch_bulk.go

@@ -32,37 +32,59 @@ type esBulkIndexerItem = docappender.BulkIndexerItem
 
 // clientLogger implements the estransport.Logger interface
 // that is required by the Elasticsearch client for logging.
-type clientLogger zap.Logger
+type clientLogger struct {
+	*zap.Logger
+	logRequestBody  bool
+	logResponseBody bool
+}
 
 // LogRoundTrip should not modify the request or response, except for consuming and closing the body.
 // Implementations have to check for nil values in request and response.
-func (cl *clientLogger) LogRoundTrip(requ *http.Request, resp *http.Response, err error, _ time.Time, dur time.Duration) error {
-	zl := (*zap.Logger)(cl)
+func (cl *clientLogger) LogRoundTrip(requ *http.Request, resp *http.Response, clientErr error, _ time.Time, dur time.Duration) error {
+	zl := cl.Logger
+
+	var fields []zap.Field
+	if cl.logRequestBody && requ != nil && requ.Body != nil {
+		if b, err := io.ReadAll(requ.Body); err == nil {
+			fields = append(fields, zap.ByteString("request_body", b))
+		}
+	}
+	if cl.logResponseBody && resp != nil && resp.Body != nil {
+		if b, err := io.ReadAll(resp.Body); err == nil {
+			fields = append(fields, zap.ByteString("response_body", b))
+		}
+	}
+
 	switch {
-	case err == nil && resp != nil:
-		zl.Debug("Request roundtrip completed.",
+	case clientErr == nil && resp != nil:
+		fields = append(
+			fields,
 			zap.String("path", sanitize.String(requ.URL.Path)),
 			zap.String("method", requ.Method),
 			zap.Duration("duration", dur),
-			zap.String("status", resp.Status))
-
-	case err != nil:
-		zl.Error("Request failed.", zap.NamedError("reason", err))
+			zap.String("status", resp.Status),
+		)
+		zl.Debug("Request roundtrip completed.", fields...)
+
+	case clientErr != nil:
+		fields = append(
+			fields,
+			zap.NamedError("reason", clientErr),
+		)
+		zl.Debug("Request failed.", fields...)
 	}
 
 	return nil
 }
 
 // RequestBodyEnabled makes the client pass a copy of request body to the logger.
-func (*clientLogger) RequestBodyEnabled() bool {
-	// TODO: introduce setting log the bodies for more detailed debug logs
-	return false
+func (cl *clientLogger) RequestBodyEnabled() bool {
+	return cl.logRequestBody
 }
 
 // ResponseBodyEnabled makes the client pass a copy of response body to the logger.
-func (*clientLogger) ResponseBodyEnabled() bool {
-	// TODO: introduce setting log the bodies for more detailed debug logs
-	return false
+func (cl *clientLogger) ResponseBodyEnabled() bool {
+	return cl.logResponseBody
 }
 
 func newElasticsearchClient(
@@ -97,6 +119,12 @@ func newElasticsearchClient(
 		return nil, err
 	}
 
+	esLogger := clientLogger{
+		Logger:          telemetry.Logger,
+		logRequestBody:  config.LogRequestBody,
+		logResponseBody: config.LogResponseBody,
+	}
+
 	return elasticsearch7.NewClient(esConfigCurrent{
 		Transport: httpClient.Transport,
 
@@ -122,7 +150,7 @@ func newElasticsearchClient(
 		// configure internal metrics reporting and logging
 		EnableMetrics:     false, // TODO
 		EnableDebugLogger: false, // TODO
-		Logger:            (*clientLogger)(telemetry.Logger),
+		Logger:            &esLogger,
 	})
 }
 

exporter/elasticsearchexporter/factory.go

@@ -84,6 +84,10 @@ func createDefaultConfig() component.Config {
 			PrefixSeparator: "-",
 			DateFormat:      "%Y.%m.%d",
 		},
+		TelemetrySettings: TelemetrySettings{
+			LogRequestBody:  false,
+			LogResponseBody: false,
+		},
 	}
 }