Skip to content

Commit 18665e9

Browse files
jmichalek132jmichalek132
committed
feat: added translation rw2 translation for gauges
Signed-off-by: Juraj Michalek <[email protected]>
1 parent 12a8df8 commit 18665e9

File tree

9 files changed

+301
-17
lines changed

9 files changed

+301
-17
lines changed

‎pkg/translator/prometheusremotewrite/helper_v2.go

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package prometheusremotewrite // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite"
5+
6+
import (
7+
"fmt"
8+
"log"
9+
"slices"
10+
11+
"github.com/prometheus/common/model"
12+
"github.com/prometheus/prometheus/model/labels"
13+
"go.opentelemetry.io/collector/pdata/pcommon"
14+
conventions "go.opentelemetry.io/collector/semconv/v1.25.0"
15+
16+
prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
17+
)
18+
19+
// createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values.
20+
// Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and
21+
// if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized.
22+
func createAttributesV2(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string,
23+
ignoreAttrs []string, logOnOverwrite bool, extras ...string) labels.Labels {
24+
resourceAttrs := resource.Attributes()
25+
serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName)
26+
instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID)
27+
28+
// Calculate the maximum possible number of labels we could return so we can preallocate l
29+
maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2
30+
31+
if haveServiceName {
32+
maxLabelCount++
33+
}
34+
35+
if haveInstanceID {
36+
maxLabelCount++
37+
}
38+
39+
// map ensures no duplicate label name
40+
l := make(map[string]string, maxLabelCount)
41+
42+
// Ensure attributes are sorted by key for consistent merging of keys which
43+
// collide when sanitized.
44+
tempSeriesLabels := labels.Labels{}
45+
// XXX: Should we always drop service namespace/service name/service instance ID from the labels
46+
// (as they get mapped to other Prometheus labels)?
47+
attributes.Range(func(key string, value pcommon.Value) bool {
48+
if !slices.Contains(ignoreAttrs, key) {
49+
tempSeriesLabels = append(tempSeriesLabels, labels.Label{Name: key, Value: value.AsString()})
50+
}
51+
return true
52+
})
53+
// TODO New returns a sorted Labels from the given labels. The caller has to guarantee that all label names are unique.
54+
seriesLabels := labels.New(tempSeriesLabels...) // This sorts by name
55+
56+
for _, label := range seriesLabels {
57+
var finalKey = prometheustranslator.NormalizeLabel(label.Name)
58+
if existingValue, alreadyExists := l[finalKey]; alreadyExists {
59+
l[finalKey] = existingValue + ";" + label.Value
60+
} else {
61+
l[finalKey] = label.Value
62+
}
63+
}
64+
65+
// Map service.name + service.namespace to job
66+
if haveServiceName {
67+
val := serviceName.AsString()
68+
if serviceNamespace, ok := resourceAttrs.Get(conventions.AttributeServiceNamespace); ok {
69+
val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val)
70+
}
71+
l[model.JobLabel] = val
72+
}
73+
// Map service.instance.id to instance
74+
if haveInstanceID {
75+
l[model.InstanceLabel] = instance.AsString()
76+
}
77+
for key, value := range externalLabels {
78+
// External labels have already been sanitized
79+
if _, alreadyExists := l[key]; alreadyExists {
80+
// Skip external labels if they are overridden by metric attributes
81+
continue
82+
}
83+
l[key] = value
84+
}
85+
86+
for i := 0; i < len(extras); i += 2 {
87+
if i+1 >= len(extras) {
88+
break
89+
}
90+
_, found := l[extras[i]]
91+
if found && logOnOverwrite {
92+
log.Println("label " + extras[i] + " is overwritten. Check if Prometheus reserved labels are used.")
93+
}
94+
// internal labels should be maintained
95+
name := extras[i]
96+
if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") {
97+
name = prometheustranslator.NormalizeLabel(name)
98+
}
99+
l[name] = extras[i+1]
100+
}
101+
102+
seriesLabels = seriesLabels[:0]
103+
for k, v := range l {
104+
seriesLabels = append(seriesLabels, labels.Label{Name: k, Value: v})
105+
}
106+
107+
return seriesLabels
108+
}

‎pkg/translator/prometheusremotewrite/helper_v2_test.go

+139
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package prometheusremotewrite
5+
6+
import (
7+
"testing"
8+
9+
"github.com/prometheus/prometheus/model/labels"
10+
"github.com/stretchr/testify/assert"
11+
"go.opentelemetry.io/collector/pdata/pcommon"
12+
)
13+
14+
// Test_createLabelSet checks resultant label names are sanitized and label in extra overrides label in labels if
15+
// collision happens. It does not check whether labels are not sorted
16+
func Test_createLabelSetV2(t *testing.T) {
17+
tests := []struct {
18+
name string
19+
resource pcommon.Resource
20+
orig pcommon.Map
21+
externalLabels map[string]string
22+
extras []string
23+
want labels.Labels
24+
}{
25+
{
26+
"labels_clean",
27+
pcommon.NewResource(),
28+
lbs1,
29+
map[string]string{},
30+
[]string{label31, value31, label32, value32},
31+
getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32),
32+
},
33+
{
34+
"labels_with_resource",
35+
func() pcommon.Resource {
36+
res := pcommon.NewResource()
37+
res.Attributes().PutStr("service.name", "prometheus")
38+
res.Attributes().PutStr("service.instance.id", "127.0.0.1:8080")
39+
return res
40+
}(),
41+
lbs1,
42+
map[string]string{},
43+
[]string{label31, value31, label32, value32},
44+
getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32, "job", "prometheus", "instance", "127.0.0.1:8080"),
45+
},
46+
{
47+
"labels_with_nonstring_resource",
48+
func() pcommon.Resource {
49+
res := pcommon.NewResource()
50+
res.Attributes().PutInt("service.name", 12345)
51+
res.Attributes().PutBool("service.instance.id", true)
52+
return res
53+
}(),
54+
lbs1,
55+
map[string]string{},
56+
[]string{label31, value31, label32, value32},
57+
getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32, "job", "12345", "instance", "true"),
58+
},
59+
{
60+
"labels_duplicate_in_extras",
61+
pcommon.NewResource(),
62+
lbs1,
63+
map[string]string{},
64+
[]string{label11, value31},
65+
getPromLabelsV2(label11, value31, label12, value12),
66+
},
67+
{
68+
"labels_dirty",
69+
pcommon.NewResource(),
70+
lbs1Dirty,
71+
map[string]string{},
72+
[]string{label31 + dirty1, value31, label32, value32},
73+
getPromLabelsV2(label11+"_", value11, "key_"+label12, value12, label31+"_", value31, label32, value32),
74+
},
75+
{
76+
"no_original_case",
77+
pcommon.NewResource(),
78+
pcommon.NewMap(),
79+
nil,
80+
[]string{label31, value31, label32, value32},
81+
getPromLabelsV2(label31, value31, label32, value32),
82+
},
83+
{
84+
"empty_extra_case",
85+
pcommon.NewResource(),
86+
lbs1,
87+
map[string]string{},
88+
[]string{"", ""},
89+
getPromLabelsV2(label11, value11, label12, value12, "", ""),
90+
},
91+
{
92+
"single_left_over_case",
93+
pcommon.NewResource(),
94+
lbs1,
95+
map[string]string{},
96+
[]string{label31, value31, label32},
97+
getPromLabelsV2(label11, value11, label12, value12, label31, value31),
98+
},
99+
{
100+
"valid_external_labels",
101+
pcommon.NewResource(),
102+
lbs1,
103+
exlbs1,
104+
[]string{label31, value31, label32, value32},
105+
getPromLabelsV2(label11, value11, label12, value12, label41, value41, label31, value31, label32, value32),
106+
},
107+
{
108+
"overwritten_external_labels",
109+
pcommon.NewResource(),
110+
lbs1,
111+
exlbs2,
112+
[]string{label31, value31, label32, value32},
113+
getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32),
114+
},
115+
{
116+
"colliding attributes",
117+
pcommon.NewResource(),
118+
lbsColliding,
119+
nil,
120+
[]string{label31, value31, label32, value32},
121+
getPromLabelsV2(collidingSanitized, value11+";"+value12, label31, value31, label32, value32),
122+
},
123+
{
124+
"sanitize_labels_starts_with_underscore",
125+
pcommon.NewResource(),
126+
lbs3,
127+
exlbs1,
128+
[]string{label31, value31, label32, value32},
129+
getPromLabelsV2(label11, value11, label12, value12, "key"+label51, value51, label41, value41, label31, value31, label32, value32),
130+
},
131+
}
132+
// run tests
133+
for _, tt := range tests {
134+
t.Run(tt.name, func(t *testing.T) {
135+
res := createAttributesV2(tt.resource, tt.orig, tt.externalLabels, nil, true, tt.extras...)
136+
assert.ElementsMatch(t, tt.want, res)
137+
})
138+
}
139+
}

‎pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ func (c *prometheusConverterV2) fromMetrics(md pmetric.Metrics, settings Setting
4949
resourceMetricsSlice := md.ResourceMetrics()
5050
for i := 0; i < resourceMetricsSlice.Len(); i++ {
5151
resourceMetrics := resourceMetricsSlice.At(i)
52+
resource := resourceMetrics.Resource()
5253
scopeMetricsSlice := resourceMetrics.ScopeMetrics()
5354
// keep track of the most recent timestamp in the ResourceMetrics for
5455
// use with the "target" info metric
@@ -77,7 +78,7 @@ func (c *prometheusConverterV2) fromMetrics(md pmetric.Metrics, settings Setting
7778
errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name()))
7879
break
7980
}
80-
c.addGaugeNumberDataPoints(dataPoints, promName)
81+
c.addGaugeNumberDataPoints(dataPoints, resource, settings, promName)
8182
case pmetric.MetricTypeSum:
8283
// TODO implement
8384
case pmetric.MetricTypeHistogram:

‎pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func TestFromMetricsV2(t *testing.T) {
2727
want := func() map[string]*writev2.TimeSeries {
2828
return map[string]*writev2.TimeSeries{
2929
"0": {
30-
LabelsRefs: []uint32{1, 2},
30+
LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8},
3131
Samples: []writev2.Sample{
3232
{Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 1.23},
3333
},

‎pkg/translator/prometheusremotewrite/number_data_points_v2.go

+12-9
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,26 @@ import (
77
"math"
88

99
"github.com/prometheus/common/model"
10-
"github.com/prometheus/prometheus/model/labels"
1110
"github.com/prometheus/prometheus/model/value"
1211
writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2"
12+
"go.opentelemetry.io/collector/pdata/pcommon"
1313
"go.opentelemetry.io/collector/pdata/pmetric"
1414
)
1515

16-
func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.NumberDataPointSlice, name string) {
16+
func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.NumberDataPointSlice,
17+
resource pcommon.Resource, settings Settings, name string) {
1718
for x := 0; x < dataPoints.Len(); x++ {
1819
pt := dataPoints.At(x)
19-
// TODO implement support for labels
2020

21-
labels := labels.Labels{
22-
labels.Label{
23-
Name: model.MetricNameLabel,
24-
Value: name,
25-
},
26-
}
21+
labels := createAttributesV2(
22+
resource,
23+
pt.Attributes(),
24+
settings.ExternalLabels,
25+
nil,
26+
true,
27+
model.MetricNameLabel,
28+
name,
29+
)
2730

2831
sample := &writev2.Sample{
2932
// convert ns to ms

‎pkg/translator/prometheusremotewrite/number_data_points_v2_test.go

+20-3
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,16 @@ func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) {
108108
for _, tt := range tests {
109109
t.Run(tt.name, func(t *testing.T) {
110110
metric := tt.metric()
111+
settings := Settings{
112+
Namespace: "",
113+
ExternalLabels: nil,
114+
DisableTargetInfo: false,
115+
ExportCreatedMetric: false,
116+
AddMetricSuffixes: false,
117+
SendMetadata: false,
118+
}
111119
converter := newPrometheusConverterV2()
112-
converter.addGaugeNumberDataPoints(metric.Gauge().DataPoints(), metric.Name())
120+
converter.addGaugeNumberDataPoints(metric.Gauge().DataPoints(), pcommon.NewResource(), settings, metric.Name())
113121
w := tt.want()
114122

115123
diff := cmp.Diff(w, converter.unique, cmpopts.EquateNaNs())
@@ -150,9 +158,18 @@ func TestPrometheusConverterV2_addGaugeNumberDataPointsDuplicate(t *testing.T) {
150158
}
151159
}
152160

161+
settings := Settings{
162+
Namespace: "",
163+
ExternalLabels: nil,
164+
DisableTargetInfo: false,
165+
ExportCreatedMetric: false,
166+
AddMetricSuffixes: false,
167+
SendMetadata: false,
168+
}
169+
153170
converter := newPrometheusConverterV2()
154-
converter.addGaugeNumberDataPoints(metric1.Gauge().DataPoints(), metric1.Name())
155-
converter.addGaugeNumberDataPoints(metric2.Gauge().DataPoints(), metric2.Name())
171+
converter.addGaugeNumberDataPoints(metric1.Gauge().DataPoints(), pcommon.NewResource(), settings, metric1.Name())
172+
converter.addGaugeNumberDataPoints(metric2.Gauge().DataPoints(), pcommon.NewResource(), settings, metric2.Name())
156173

157174
assert.Equal(t, want(), converter.unique)
158175

‎pkg/translator/prometheusremotewrite/testutils_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"testing"
1111
"time"
1212

13+
"github.com/prometheus/prometheus/model/labels"
1314
"github.com/prometheus/prometheus/prompb"
1415
"github.com/stretchr/testify/require"
1516
"go.opentelemetry.io/collector/pdata/pcommon"
@@ -136,6 +137,18 @@ func getAttributes(labels ...string) pcommon.Map {
136137
return attributeMap
137138
}
138139

140+
// Prometheus TimeSeries
141+
func getPromLabelsV2(lbs ...string) labels.Labels {
142+
pbLbs := labels.Labels{}
143+
for i := 0; i < len(lbs); i += 2 {
144+
pbLbs = append(pbLbs, labels.Label{
145+
Name: lbs[i],
146+
Value: lbs[i+1],
147+
})
148+
}
149+
return pbLbs
150+
}
151+
139152
// Prometheus TimeSeries
140153
func getPromLabels(lbs ...string) []prompb.Label {
141154
pbLbs := prompb.Labels{

‎receiver/cloudflarereceiver/go.mod

+3-3
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,11 @@ require (
4949
go.opentelemetry.io/collector/pdata/pprofile v0.111.1-0.20241008154146-ea48c09c31ae // indirect
5050
go.opentelemetry.io/collector/pipeline v0.111.1-0.20241008154146-ea48c09c31ae // indirect
5151
go.opentelemetry.io/collector/receiver/receiverprofiles v0.111.1-0.20241008154146-ea48c09c31ae // indirect
52-
go.opentelemetry.io/otel v1.30.0 // indirect
53-
go.opentelemetry.io/otel/metric v1.30.0 // indirect
52+
go.opentelemetry.io/otel v1.31.0 // indirect
53+
go.opentelemetry.io/otel/metric v1.31.0 // indirect
5454
go.opentelemetry.io/otel/sdk v1.30.0 // indirect
5555
go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect
56-
go.opentelemetry.io/otel/trace v1.30.0 // indirect
56+
go.opentelemetry.io/otel/trace v1.31.0 // indirect
5757
golang.org/x/net v0.28.0 // indirect
5858
golang.org/x/sys v0.25.0 // indirect
5959
golang.org/x/text v0.19.0 // indirect

0 commit comments

Comments
 (0)