Write created lines when negotiating OpenMetrics (#504)

* expfmt/openmetrics: Write created timestamps for counters, summaries and histograms
* expfmt/encoder: Allow opt-in for OM created lines

---------

Signed-off-by: Arthur Silva Sens <[email protected]>

Author: Arthur Silva Sens

expfmt/encode.go

@@ -139,7 +139,13 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format {
 // interface is kept for backwards compatibility.
 // In cases where the Format does not allow for UTF-8 names, the global
 // NameEscapingScheme will be applied.
-func NewEncoder(w io.Writer, format Format) Encoder {
+//
+// NewEncoder can be called with additional options to customize the OpenMetrics text output.
+// For example:
+// NewEncoder(w, FmtOpenMetrics_1_0_0, WithCreatedLines())
+//
+// Extra options are ignored for all other formats.
+func NewEncoder(w io.Writer, format Format, options ...EncoderOption) Encoder {
 	escapingScheme := format.ToEscapingScheme()
 
 	switch format.FormatType() {
@@ -178,7 +184,7 @@ func NewEncoder(w io.Writer, format Format) Encoder {
 	case TypeOpenMetrics:
 		return encoderCloser{
 			encode: func(v *dto.MetricFamily) error {
-				_, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme))
+				_, err := MetricFamilyToOpenMetrics(w, model.EscapeMetricFamily(v, escapingScheme), options...)
 				return err
 			},
 			close: func() error {

expfmt/openmetrics_create.go

@@ -22,11 +22,35 @@ import (
 	"strconv"
 	"strings"
 
+	"google.golang.org/protobuf/types/known/timestamppb"
+
 	"github.com/prometheus/common/model"
 
 	dto "github.com/prometheus/client_model/go"
 )
 
+type encoderOption struct {
+	withCreatedLines bool
+}
+
+type EncoderOption func(*encoderOption)
+
+// WithCreatedLines is an EncoderOption that configures the OpenMetrics encoder
+// to include _created lines (See
+// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#counter-1).
+// Created timestamps can improve the accuracy of series reset detection, but
+// come with a bandwidth cost.
+//
+// At the time of writing, created timestamp ingestion is still experimental in
+// Prometheus and need to be enabled with the feature-flag
+// `--feature-flag=created-timestamp-zero-ingestion`, and breaking changes are
+// still possible. Therefore, it is recommended to use this feature with caution.
+func WithCreatedLines() EncoderOption {
+	return func(t *encoderOption) {
+		t.withCreatedLines = true
+	}
+}
+
 // MetricFamilyToOpenMetrics converts a MetricFamily proto message into the
 // OpenMetrics text format and writes the resulting lines to 'out'. It returns
 // the number of bytes written and any error encountered. The output will have
@@ -64,15 +88,20 @@ import (
 //     its type will be set to `unknown` in that case to avoid invalid OpenMetrics
 //     output.
 //
-//   - No support for the following (optional) features: `# UNIT` line, `_created`
-//     line, info type, stateset type, gaugehistogram type.
+//   - No support for the following (optional) features: `# UNIT` line, info type,
+//     stateset type, gaugehistogram type.
 //
 //   - The size of exemplar labels is not checked (i.e. it's possible to create
 //     exemplars that are larger than allowed by the OpenMetrics specification).
 //
 //   - The value of Counters is not checked. (OpenMetrics doesn't allow counters
 //     with a `NaN` value.)
-func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int, err error) {
+func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily, options ...EncoderOption) (written int, err error) {
+	toOM := encoderOption{}
+	for _, option := range options {
+		option(&toOM)
+	}
+
 	name := in.GetName()
 	if name == "" {
 		return 0, fmt.Errorf("MetricFamily has no name: %s", in)
@@ -164,6 +193,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
 		return
 	}
 
+	var createdTsBytesWritten int
 	// Finally the samples, one line for each.
 	for _, metric := range in.Metric {
 		switch metricType {
@@ -181,6 +211,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
 				metric.Counter.GetValue(), 0, false,
 				metric.Counter.Exemplar,
 			)
+			if toOM.withCreatedLines && metric.Counter.CreatedTimestamp != nil {
+				createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "_total", metric, "", 0, metric.Counter.GetCreatedTimestamp())
+				n += createdTsBytesWritten
+			}
 		case dto.MetricType_GAUGE:
 			if metric.Gauge == nil {
 				return written, fmt.Errorf(
@@ -235,6 +269,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
 				0, metric.Summary.GetSampleCount(), true,
 				nil,
 			)
+			if toOM.withCreatedLines && metric.Summary.CreatedTimestamp != nil {
+				createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "", metric, "", 0, metric.Summary.GetCreatedTimestamp())
+				n += createdTsBytesWritten
+			}
 		case dto.MetricType_HISTOGRAM:
 			if metric.Histogram == nil {
 				return written, fmt.Errorf(
@@ -283,6 +321,10 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int
 				0, metric.Histogram.GetSampleCount(), true,
 				nil,
 			)
+			if toOM.withCreatedLines && metric.Histogram.CreatedTimestamp != nil {
+				createdTsBytesWritten, err = writeOpenMetricsCreated(w, name, "", metric, "", 0, metric.Histogram.GetCreatedTimestamp())
+				n += createdTsBytesWritten
+			}
 		default:
 			return written, fmt.Errorf(
 				"unexpected type in metric %s %s", name, metric,
@@ -473,6 +515,49 @@ func writeOpenMetricsNameAndLabelPairs(
 	return written, nil
 }
 
+// writeOpenMetricsCreated writes the created timestamp for a single time series
+// following OpenMetrics text format to w, given the metric name, the metric proto
+// message itself, optionally a suffix to be removed, e.g. '_total' for counters,
+// an additional label name with a float64 value (use empty string as label name if
+// not required) and the timestamp that represents the created timestamp.
+// The function returns the number of bytes written and any error encountered.
+func writeOpenMetricsCreated(w enhancedWriter,
+	name, suffixToTrim string, metric *dto.Metric,
+	additionalLabelName string, additionalLabelValue float64,
+	createdTimestamp *timestamppb.Timestamp,
+) (int, error) {
+	written := 0
+	n, err := writeOpenMetricsNameAndLabelPairs(
+		w, strings.TrimSuffix(name, suffixToTrim)+"_created", metric.Label, additionalLabelName, additionalLabelValue,
+	)
+	written += n
+	if err != nil {
+		return written, err
+	}
+
+	err = w.WriteByte(' ')
+	written++
+	if err != nil {
+		return written, err
+	}
+
+	// TODO(beorn7): Format this directly from components of ts to
+	// avoid overflow/underflow and precision issues of the float
+	// conversion.
+	n, err = writeOpenMetricsFloat(w, float64(createdTimestamp.AsTime().UnixNano())/1e9)
+	written += n
+	if err != nil {
+		return written, err
+	}
+
+	err = w.WriteByte('\n')
+	written++
+	if err != nil {
+		return written, err
+	}
+	return written, nil
+}
+
 // writeExemplar writes the provided exemplar in OpenMetrics format to w. The
 // function returns the number of bytes written and any error encountered.
 func writeExemplar(w enhancedWriter, e *dto.Exemplar) (int, error) {

expfmt/openmetrics_create_test.go

@@ -41,8 +41,9 @@ func TestCreateOpenMetrics(t *testing.T) {
 	}()
 
 	scenarios := []struct {
-		in  *dto.MetricFamily
-		out string
+		in      *dto.MetricFamily
+		options []EncoderOption
+		out     string
 	}{
 		// 0: Counter, timestamp given, no _total suffix.
 		{
@@ -306,6 +307,7 @@ unknown_name{name_1="value 1"} -1.23e-45
 									Value:    proto.Float64(0),
 								},
 							},
+							CreatedTimestamp: openMetricsTimestamp,
 						},
 					},
 					{
@@ -336,22 +338,26 @@ unknown_name{name_1="value 1"} -1.23e-45
 									Value:    proto.Float64(3),
 								},
 							},
+							CreatedTimestamp: openMetricsTimestamp,
 						},
 					},
 				},
 			},
+			options: []EncoderOption{WithCreatedLines()},
 			out: `# HELP summary_name summary docstring
 # TYPE summary_name summary
 summary_name{quantile="0.5"} -1.23
 summary_name{quantile="0.9"} 0.2342354
 summary_name{quantile="0.99"} 0.0
 summary_name_sum -3.4567
 summary_name_count 42
+summary_name_created 12345.6
 summary_name{name_1="value 1",name_2="value 2",quantile="0.5"} 1.0
 summary_name{name_1="value 1",name_2="value 2",quantile="0.9"} 2.0
 summary_name{name_1="value 1",name_2="value 2",quantile="0.99"} 3.0
 summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971
 summary_name_count{name_1="value 1",name_2="value 2"} 4711
+summary_name_created{name_1="value 1",name_2="value 2"} 12345.6
 `,
 		},
 		// 7: Histogram
@@ -387,10 +393,12 @@ summary_name_count{name_1="value 1",name_2="value 2"} 4711
 									CumulativeCount: proto.Uint64(2693),
 								},
 							},
+							CreatedTimestamp: openMetricsTimestamp,
 						},
 					},
 				},
 			},
+			options: []EncoderOption{WithCreatedLines()},
 			out: `# HELP request_duration_microseconds The response latency.
 # TYPE request_duration_microseconds histogram
 request_duration_microseconds_bucket{le="100.0"} 123
@@ -400,6 +408,7 @@ request_duration_microseconds_bucket{le="172.8"} 1524
 request_duration_microseconds_bucket{le="+Inf"} 2693
 request_duration_microseconds_sum 1.7560473e+06
 request_duration_microseconds_count 2693
+request_duration_microseconds_created 12345.6
 `,
 		},
 		// 8: Histogram with missing +Inf bucket.
@@ -522,7 +531,30 @@ request_duration_microseconds_count 2693
 				Metric: []*dto.Metric{
 					{
 						Counter: &dto.Counter{
-							Value: proto.Float64(42),
+							Value:            proto.Float64(42),
+							CreatedTimestamp: openMetricsTimestamp,
+						},
+					},
+				},
+			},
+			options: []EncoderOption{WithCreatedLines()},
+			out: `# HELP foos Number of foos.
+# TYPE foos counter
+foos_total 42.0
+foos_created 12345.6
+`,
+		},
+		// 11: Simple Counter without created line.
+		{
+			in: &dto.MetricFamily{
+				Name: proto.String("foos_total"),
+				Help: proto.String("Number of foos."),
+				Type: dto.MetricType_COUNTER.Enum(),
+				Metric: []*dto.Metric{
+					{
+						Counter: &dto.Counter{
+							Value:            proto.Float64(42),
+							CreatedTimestamp: openMetricsTimestamp,
 						},
 					},
 				},
@@ -532,7 +564,7 @@ request_duration_microseconds_count 2693
 foos_total 42.0
 `,
 		},
-		// 11: No metric.
+		// 12: No metric.
 		{
 			in: &dto.MetricFamily{
 				Name:   proto.String("name_total"),
@@ -573,7 +605,7 @@ foos_total 42.0
 
 	for i, scenario := range scenarios {
 		out := bytes.NewBuffer(make([]byte, 0, len(scenario.out)))
-		n, err := MetricFamilyToOpenMetrics(out, scenario.in)
+		n, err := MetricFamilyToOpenMetrics(out, scenario.in, scenario.options...)
 		if err != nil {
 			t.Errorf("%d. error: %s", i, err)
 			continue