Skip to content

Commit e3fccb7

Browse files
jtamajtama
authored
[exporter/loki] Automatically maps severity value to "level" attribute (#14560)
Automatically maps severity value to "level" attribute
1 parent a2658ad commit e3fccb7

File tree

4 files changed

+148
-12
lines changed

4 files changed

+148
-12
lines changed

‎.chloggen/14313-loki-convert.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: enhancement
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
5+
component: exporter/loki
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: Automatic mapping beetwen `LogRecord.SeverityNumber` to `LogRecord.Attributes["level"]`
9+
10+
# One or more tracking issues related to the change
11+
issues: [14313]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext:

‎exporter/lokiexporter/README.md

+4
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,10 @@ by tenant and send requests with the `X-Scope-OrgID` header set to relevant tena
113113
If the `loki.tenant` hint attribute is present in both resource or log attributes,
114114
then the look-up for a tenant value from resource attributes takes precedence.
115115

116+
## Severity
117+
118+
OpenTelemetry uses `record.severity` to track log levels where loki uses `record.attributes.level` for the same. The exporter automatically maps the two, except if a "level" attribute already exists.
119+
116120
## Advanced Configuration
117121

118122
Several helper files are leveraged to provide additional capabilities automatically:

‎pkg/translator/loki/logs_to_loki.go

+58
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package loki // import "github.com/open-telemetry/opentelemetry-collector-contri
1616

1717
import (
1818
"fmt"
19+
"strings"
1920

2021
"github.com/grafana/loki/pkg/logproto"
2122
"go.opentelemetry.io/collector/pdata/pcommon"
@@ -34,6 +35,10 @@ type PushReport struct {
3435
NumDropped int
3536
}
3637

38+
const (
39+
levelAttributeName = "level"
40+
)
41+
3742
// LogsToLokiRequests converts a Logs pipeline data into Loki PushRequests grouped
3843
// by tenant. The tenant value is inferred from the `loki.tenant` resource or log
3944
// attribute hint. If the `loki.tenant` attribute is present in both resource or
@@ -70,6 +75,9 @@ func LogsToLokiRequests(ld plog.Logs) map[string]PushRequest {
7075
resource := pcommon.NewResource()
7176
rls.At(i).Resource().CopyTo(resource)
7277

78+
// adds level attribute from log.severityNumber
79+
addLogLevelAttributeAndHint(log)
80+
7381
// resolve tenant and get/create a push request group
7482
tenant := getTenantFromTenantHint(log.Attributes(), resource.Attributes())
7583
group, ok := groups[tenant]
@@ -207,6 +215,9 @@ func LogsToLoki(ld plog.Logs) (*logproto.PushRequest, *PushReport) {
207215
resource := pcommon.NewResource()
208216
rls.At(i).Resource().CopyTo(resource)
209217

218+
// adds level attribute from log.severityNumber
219+
addLogLevelAttributeAndHint(log)
220+
210221
format := getFormatFromFormatHint(log.Attributes(), resource.Attributes())
211222

212223
mergedLabels := convertAttributesAndMerge(log.Attributes(), resource.Attributes())
@@ -252,3 +263,50 @@ func LogsToLoki(ld plog.Logs) (*logproto.PushRequest, *PushReport) {
252263

253264
return pr, report
254265
}
266+
267+
func addLogLevelAttributeAndHint(log plog.LogRecord) {
268+
if log.SeverityNumber() == plog.SeverityNumberUnspecified {
269+
return
270+
}
271+
addHint(log)
272+
if _, found := log.Attributes().Get(levelAttributeName); !found {
273+
level := severityNumberToLevel[log.SeverityNumber().String()]
274+
log.Attributes().PutStr(levelAttributeName, level)
275+
}
276+
}
277+
278+
func addHint(log plog.LogRecord) {
279+
if value, found := log.Attributes().Get(hintAttributes); found && !strings.Contains(value.AsString(), levelAttributeName) {
280+
log.Attributes().PutStr(hintAttributes, fmt.Sprintf("%s,%s", value.AsString(), levelAttributeName))
281+
} else {
282+
log.Attributes().PutStr(hintAttributes, levelAttributeName)
283+
}
284+
}
285+
286+
var severityNumberToLevel = map[string]string{
287+
plog.SeverityNumberUnspecified.String(): "UNSPECIFIED",
288+
plog.SeverityNumberTrace.String(): "TRACE",
289+
plog.SeverityNumberTrace2.String(): "TRACE2",
290+
plog.SeverityNumberTrace3.String(): "TRACE3",
291+
plog.SeverityNumberTrace4.String(): "TRACE4",
292+
plog.SeverityNumberDebug.String(): "DEBUG",
293+
plog.SeverityNumberDebug2.String(): "DEBUG2",
294+
plog.SeverityNumberDebug3.String(): "DEBUG3",
295+
plog.SeverityNumberDebug4.String(): "DEBUG4",
296+
plog.SeverityNumberInfo.String(): "INFO",
297+
plog.SeverityNumberInfo2.String(): "INFO2",
298+
plog.SeverityNumberInfo3.String(): "INFO3",
299+
plog.SeverityNumberInfo4.String(): "INFO4",
300+
plog.SeverityNumberWarn.String(): "WARN",
301+
plog.SeverityNumberWarn2.String(): "WARN2",
302+
plog.SeverityNumberWarn3.String(): "WARN3",
303+
plog.SeverityNumberWarn4.String(): "WARN4",
304+
plog.SeverityNumberError.String(): "ERROR",
305+
plog.SeverityNumberError2.String(): "ERROR2",
306+
plog.SeverityNumberError3.String(): "ERROR3",
307+
plog.SeverityNumberError4.String(): "ERROR4",
308+
plog.SeverityNumberFatal.String(): "FATAL",
309+
plog.SeverityNumberFatal2.String(): "FATAL2",
310+
plog.SeverityNumberFatal3.String(): "FATAL3",
311+
plog.SeverityNumberFatal4.String(): "FATAL4",
312+
}

‎pkg/translator/loki/logs_to_loki_test.go

+70-12
Original file line numberDiff line numberDiff line change
@@ -244,12 +244,14 @@ func TestLogsToLokiRequestWithGroupingByTenant(t *testing.T) {
244244

245245
func TestLogsToLokiRequestWithoutTenant(t *testing.T) {
246246
testCases := []struct {
247-
desc string
248-
hints map[string]interface{}
249-
attrs map[string]interface{}
250-
res map[string]interface{}
251-
expectedLabel string
252-
expectedLines []string
247+
desc string
248+
hints map[string]interface{}
249+
attrs map[string]interface{}
250+
res map[string]interface{}
251+
severity plog.SeverityNumber
252+
levelAttribute string
253+
expectedLabel string
254+
expectedLines []string
253255
}{
254256
{
255257
desc: "with attribute to label and regular attribute",
@@ -300,6 +302,27 @@ func TestLogsToLokiRequestWithoutTenant(t *testing.T) {
300302
`traceID=03000000000000000000000000000000 attribute_http.status=200`,
301303
},
302304
},
305+
{
306+
desc: "with severity to label",
307+
severity: plog.SeverityNumberDebug4,
308+
expectedLabel: `{exporter="OTLP", level="DEBUG4"}`,
309+
expectedLines: []string{
310+
`{"traceid":"01000000000000000000000000000000"}`,
311+
`{"traceid":"02000000000000000000000000000000"}`,
312+
`{"traceid":"03000000000000000000000000000000"}`,
313+
},
314+
},
315+
{
316+
desc: "with severity, already existing level",
317+
severity: plog.SeverityNumberDebug4,
318+
levelAttribute: "dummy",
319+
expectedLabel: `{exporter="OTLP", level="dummy"}`,
320+
expectedLines: []string{
321+
`{"traceid":"01000000000000000000000000000000"}`,
322+
`{"traceid":"02000000000000000000000000000000"}`,
323+
`{"traceid":"03000000000000000000000000000000"}`,
324+
},
325+
},
303326
}
304327
for _, tt := range testCases {
305328
t.Run(tt.desc, func(t *testing.T) {
@@ -310,6 +333,10 @@ func TestLogsToLokiRequestWithoutTenant(t *testing.T) {
310333
ld.ResourceLogs().At(0).ScopeLogs().AppendEmpty()
311334
ld.ResourceLogs().At(0).ScopeLogs().At(i).LogRecords().AppendEmpty()
312335
ld.ResourceLogs().At(0).ScopeLogs().At(i).LogRecords().At(0).SetTraceID([16]byte{byte(i + 1)})
336+
ld.ResourceLogs().At(0).ScopeLogs().At(i).LogRecords().At(0).SetSeverityNumber(tt.severity)
337+
if len(tt.levelAttribute) > 0 {
338+
ld.ResourceLogs().At(0).ScopeLogs().At(i).LogRecords().At(0).Attributes().PutStr(levelAttributeName, tt.levelAttribute)
339+
}
313340
}
314341

315342
if len(tt.res) > 0 {
@@ -355,12 +382,14 @@ func TestLogsToLokiRequestWithoutTenant(t *testing.T) {
355382

356383
func TestLogsToLoki(t *testing.T) {
357384
testCases := []struct {
358-
desc string
359-
hints map[string]interface{}
360-
attrs map[string]interface{}
361-
res map[string]interface{}
362-
expectedLabel string
363-
expectedLines []string
385+
desc string
386+
hints map[string]interface{}
387+
attrs map[string]interface{}
388+
res map[string]interface{}
389+
severity plog.SeverityNumber
390+
levelAttribute string
391+
expectedLabel string
392+
expectedLines []string
364393
}{
365394
{
366395
desc: "with attribute to label and regular attribute",
@@ -411,6 +440,27 @@ func TestLogsToLoki(t *testing.T) {
411440
`traceID=01020304050600000000000000000000 resource_region.az=eu-west-1a`,
412441
},
413442
},
443+
{
444+
desc: "with severity to label",
445+
severity: plog.SeverityNumberDebug4,
446+
expectedLabel: `{exporter="OTLP", level="DEBUG4"}`,
447+
expectedLines: []string{
448+
`{"traceid":"01020304000000000000000000000000"}`,
449+
`{"traceid":"01020304050000000000000000000000"}`,
450+
`{"traceid":"01020304050600000000000000000000"}`,
451+
},
452+
},
453+
{
454+
desc: "with severity, already existing level",
455+
severity: plog.SeverityNumberDebug4,
456+
levelAttribute: "dummy",
457+
expectedLabel: `{exporter="OTLP", level="dummy"}`,
458+
expectedLines: []string{
459+
`{"traceid":"01020304000000000000000000000000"}`,
460+
`{"traceid":"01020304050000000000000000000000"}`,
461+
`{"traceid":"01020304050600000000000000000000"}`,
462+
},
463+
},
414464
}
415465
for _, tC := range testCases {
416466
t.Run(tC.desc, func(t *testing.T) {
@@ -421,8 +471,11 @@ func TestLogsToLoki(t *testing.T) {
421471
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().AppendEmpty()
422472
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().AppendEmpty()
423473
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().AppendEmpty()
474+
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).SetSeverityNumber(tC.severity)
424475
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).SetTraceID(pcommon.TraceID([16]byte{1, 2, 3, 4}))
476+
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).SetSeverityNumber(tC.severity)
425477
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).SetTraceID(pcommon.TraceID([16]byte{1, 2, 3, 4, 5}))
478+
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).SetSeverityNumber(tC.severity)
426479
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).SetTraceID(pcommon.TraceID([16]byte{1, 2, 3, 4, 5, 6}))
427480

428481
// copy the attributes from the test case to the log entry
@@ -434,6 +487,11 @@ func TestLogsToLoki(t *testing.T) {
434487
if len(tC.res) > 0 {
435488
ld.ResourceLogs().At(0).Resource().Attributes().FromRaw(tC.res)
436489
}
490+
if len(tC.levelAttribute) > 0 {
491+
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().PutStr(levelAttributeName, tC.levelAttribute)
492+
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(1).Attributes().PutStr(levelAttributeName, tC.levelAttribute)
493+
ld.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(2).Attributes().PutStr(levelAttributeName, tC.levelAttribute)
494+
}
437495

438496
// we can't use copy here, as the value (Value) will be used as string lookup later, so, we need to convert it to string now
439497
for k, v := range tC.hints {

0 commit comments

Comments
 (0)