[exporter/prometheusremotewrite] : WAL write success/failure telemetry (open-telemetry#39843)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

This PR introduces Write and Write failures metric from which we can
derive the success ratio or the failure ratio.
- `otelcol_exporter_prometheusremotewrite_wal_writes`: Total WAL writes
requests
- `otelcol_exporter_prometheusremotewrite_wal_writes_failures`: Total
WAL write failures


I decided to introduce the code in the handle export function of the
exporter , just before calling the `wal.persistToWAL`

<!-- Issue number (e.g. #1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Part of  open-telemetry#39556 

<!--Describe what testing was performed and which tests were added.-->
#### Testing
[
![Screenshot 2025-05-09 at 15 34
45](https://github.com/user-attachments/assets/4489b13a-a538-40ef-9ff7-de6d9f23290a)
](url)


<!--Describe the documentation added.-->
#### Documentation

<!--Please delete paragraphs that you did not use before submitting.-->

---------

Co-authored-by: Arthur Silva Sens <[email protected]>

Author: 2 people

.chloggen/nickange_prometheusremotewriteexporter_feat-add-wal-observability.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: prometheusremotewriteexproter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: |
+  Adds wal metrics to the Prometheus Remote WRite Exporter. The new metrics are:
+  - `otelcol_exporter_prometheusremotewrite_wal_writes`: The total number of WAL writes.
+  - `otelcol_exporter_prometheusremotewrite_wal_writes_failures`: The total number of WAL write failures.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [39556]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/prometheusremotewriteexporter/README.md

@@ -177,4 +177,4 @@ Out-of-order support in Prometheus must be enabled for multiple consumers.
 This can be done by using the `tsdb.out_of_order_time_window: 10m` settings. Please choose an appropriate time window to support pushing the worst-case scenarios of a "queue" build-up on the sender side.
 
 See for more info:
-- https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb
+- https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb

exporter/prometheusremotewriteexporter/documentation.md

@@ -37,3 +37,19 @@ Number of Prometheus time series that were translated from OTel metrics
 | Unit | Metric Type | Value Type | Monotonic |
 | ---- | ----------- | ---------- | --------- |
 | 1 | Sum | Int | true |
+
+### otelcol_exporter_prometheusremotewrite_wal_writes
+
+Number of WAL writes
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |
+
+### otelcol_exporter_prometheusremotewrite_wal_writes_failures
+
+Number of WAL writes that failed
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| 1 | Sum | Int | true |

exporter/prometheusremotewriteexporter/exporter.go

@@ -135,7 +135,7 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
 		return nil, err
 	}
 
-	if err := config.RemoteWriteProtoMsg.Validate(cfg.RemoteWriteProtoMsg); err != nil {
+	if err = config.RemoteWriteProtoMsg.Validate(cfg.RemoteWriteProtoMsg); err != nil {
 		return nil, err
 	}
 
@@ -177,7 +177,10 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
 
 	prwe.settings.Logger.Info("starting prometheus remote write exporter", zap.Any("ProtoMsg", cfg.RemoteWriteProtoMsg))
 
-	prwe.wal = newWAL(cfg.WAL, prwe.export)
+	prwe.wal, err = newWAL(cfg.WAL, set, prwe.export)
+	if err != nil {
+		return nil, err
+	}
 	return prwe, nil
 }
 
@@ -288,8 +291,10 @@ func (prwe *prwExporter) handleExport(ctx context.Context, tsMap map[string]*pro
 	}
 
 	// Otherwise the WAL is enabled, and just persist the requests to the WAL
-	// and they'll be exported in another goroutine to the RemoteWrite endpoint.
-	if err = prwe.wal.persistToWAL(requests); err != nil {
+	prwe.wal.telemetry.recordWALWrites(ctx)
+	err = prwe.wal.persistToWAL(requests)
+	if err != nil {
+		prwe.wal.telemetry.recordWALWritesFailures(ctx)
 		return consumererror.NewPermanent(err)
 	}
 	return nil

exporter/prometheusremotewriteexporter/internal/metadata/generated_telemetry.go

@@ -46,3 +46,17 @@ telemetry:
       sum:
         value_type: int
         monotonic: true
+    exporter_prometheusremotewrite_wal_writes:
+      enabled: true
+      description: Number of WAL writes
+      unit: "1"
+      sum:
+        value_type: int
+        monotonic: true
+    exporter_prometheusremotewrite_wal_writes_failures:
+      enabled: true
+      description: Number of WAL writes that failed
+      unit: "1"
+      sum:
+        value_type: int
+        monotonic: true

exporter/prometheusremotewriteexporter/internal/metadatatest/generated_telemetrytest.go

@@ -15,10 +15,44 @@ import (
 	"github.com/gogo/protobuf/proto"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/tidwall/wal"
+	"go.opentelemetry.io/collector/exporter"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter/internal/metadata"
 )
 
+type prwWalTelemetry interface {
+	recordWALWrites(ctx context.Context)
+	recordWALWritesFailures(ctx context.Context)
+}
+
+type prwWalTelemetryOTel struct {
+	telemetryBuilder *metadata.TelemetryBuilder
+	otelAttrs        []attribute.KeyValue
+}
+
+func (p *prwWalTelemetryOTel) recordWALWrites(ctx context.Context) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWalWrites.Add(ctx, 1, metric.WithAttributes(p.otelAttrs...))
+}
+
+func (p *prwWalTelemetryOTel) recordWALWritesFailures(ctx context.Context) {
+	p.telemetryBuilder.ExporterPrometheusremotewriteWalWritesFailures.Add(ctx, 1, metric.WithAttributes(p.otelAttrs...))
+}
+
+func newPRWWalTelemetry(set exporter.Settings) (prwWalTelemetry, error) {
+	telemetryBuilder, err := metadata.NewTelemetryBuilder(set.TelemetrySettings)
+	if err != nil {
+		return nil, err
+	}
+	return &prwWalTelemetryOTel{
+		telemetryBuilder: telemetryBuilder,
+		otelAttrs:        []attribute.KeyValue{},
+	}, nil
+}
+
 type prweWAL struct {
 	wg        sync.WaitGroup // wg waits for the go routines to finish.
 	mu        sync.Mutex     // mu protects the fields below.
@@ -33,6 +67,8 @@ type prweWAL struct {
 	rNotify   chan struct{}
 	rWALIndex *atomic.Uint64
 	wWALIndex *atomic.Uint64
+
+	telemetry prwWalTelemetry
 }
 
 const (
@@ -60,11 +96,16 @@ func (wc *WALConfig) truncateFrequency() time.Duration {
 	return defaultWALTruncateFrequency
 }
 
-func newWAL(walConfig *WALConfig, exportSink func(context.Context, []*prompb.WriteRequest) error) *prweWAL {
+func newWAL(walConfig *WALConfig, set exporter.Settings, exportSink func(context.Context, []*prompb.WriteRequest) error) (*prweWAL, error) {
 	if walConfig == nil {
 		// There are cases for which the WAL can be disabled.
 		// TODO: Perhaps log that the WAL wasn't enabled.
-		return nil
+		return nil, nil
+	}
+
+	telemetryPRWWal, err := newPRWWalTelemetry(set)
+	if err != nil {
+		return nil, err
 	}
 
 	return &prweWAL{
@@ -74,7 +115,8 @@ func newWAL(walConfig *WALConfig, exportSink func(context.Context, []*prompb.Wri
 		rNotify:    make(chan struct{}),
 		rWALIndex:  &atomic.Uint64{},
 		wWALIndex:  &atomic.Uint64{},
-	}
+		telemetry:  telemetryPRWWal,
+	}, nil
 }
 
 func (wc *WALConfig) createWAL() (*wal.Log, string, error) {
@@ -323,6 +365,7 @@ func (prweWAL *prweWAL) persistToWAL(requests []*prompb.WriteRequest) error {
 	case prweWAL.rNotify <- struct{}{}:
 	default:
 	}
+
 	return prweWAL.wal.WriteBatch(batch)
 }