Skip to content

Commit a7b781e

Browse files
wph95wph95
authored
all builtin parser add mapstructure support (#58)
* Add mapstructure support to parsers
1 parent 83ae561 commit a7b781e

File tree

20 files changed

+366
-55
lines changed

20 files changed

+366
-55
lines changed

‎operator/builtin/parser/json/json.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func NewJSONParserConfig(operatorID string) *JSONParserConfig {
3737

3838
// JSONParserConfig is the configuration of a JSON parser operator.
3939
type JSONParserConfig struct {
40-
helper.ParserConfig `yaml:",inline"`
40+
helper.ParserConfig `mapstructure:",squash" yaml:",inline"`
4141
}
4242

4343
// Build will build a JSON parser operator.

‎operator/builtin/parser/json/json_test.go

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,15 @@ import (
2020
"time"
2121

2222
jsoniter "github.com/json-iterator/go"
23+
"github.com/stretchr/testify/mock"
24+
"github.com/stretchr/testify/require"
25+
"go.uber.org/zap"
26+
"gopkg.in/yaml.v2"
27+
2328
"github.com/open-telemetry/opentelemetry-log-collection/entry"
2429
"github.com/open-telemetry/opentelemetry-log-collection/operator"
2530
"github.com/open-telemetry/opentelemetry-log-collection/operator/helper"
2631
"github.com/open-telemetry/opentelemetry-log-collection/testutil"
27-
28-
"github.com/stretchr/testify/mock"
29-
"github.com/stretchr/testify/require"
30-
"go.uber.org/zap"
3132
)
3233

3334
func newTestParser(t *testing.T) *JSONParser {
@@ -231,3 +232,35 @@ func TestJSONParserWithEmbeddedTimeParser(t *testing.T) {
231232
})
232233
}
233234
}
235+
236+
func TestJsonParserConfig(t *testing.T) {
237+
expect := NewJSONParserConfig("test")
238+
expect.ParseFrom = entry.NewRecordField("from")
239+
expect.ParseTo = entry.NewRecordField("to")
240+
241+
t.Run("mapstructure", func(t *testing.T) {
242+
input := map[string]interface{}{
243+
"id": "test",
244+
"type": "json_parser",
245+
"parse_from": "$.from",
246+
"parse_to": "$.to",
247+
"on_error": "send",
248+
}
249+
var actual JSONParserConfig
250+
err := helper.UnmarshalMapstructure(input, &actual)
251+
require.NoError(t, err)
252+
require.Equal(t, expect, &actual)
253+
})
254+
255+
t.Run("yaml", func(t *testing.T) {
256+
input := `type: json_parser
257+
id: test
258+
on_error: "send"
259+
parse_from: $.from
260+
parse_to: $.to`
261+
var actual JSONParserConfig
262+
err := yaml.Unmarshal([]byte(input), &actual)
263+
require.NoError(t, err)
264+
require.Equal(t, expect, &actual)
265+
})
266+
}

‎operator/builtin/parser/regex/regex.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ func NewRegexParserConfig(operatorID string) *RegexParserConfig {
3838

3939
// RegexParserConfig is the configuration of a regex parser operator.
4040
type RegexParserConfig struct {
41-
helper.ParserConfig `yaml:",inline"`
41+
helper.ParserConfig `mapstructure:",squash" yaml:",inline"`
4242

43-
Regex string `json:"regex" yaml:"regex"`
43+
Regex string `mapstructure:"regex" json:"regex" yaml:"regex"`
4444
}
4545

4646
// Build will build a regex parser operator.

‎operator/builtin/parser/regex/regex_test.go

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@ import (
1818
"context"
1919
"testing"
2020

21+
"github.com/stretchr/testify/require"
22+
"gopkg.in/yaml.v2"
23+
2124
"github.com/open-telemetry/opentelemetry-log-collection/entry"
2225
"github.com/open-telemetry/opentelemetry-log-collection/operator"
26+
"github.com/open-telemetry/opentelemetry-log-collection/operator/helper"
2327
"github.com/open-telemetry/opentelemetry-log-collection/testutil"
24-
"github.com/stretchr/testify/require"
2528
)
2629

2730
func newTestParser(t *testing.T, regex string) *RegexParser {
@@ -158,3 +161,39 @@ func TestBuildParserRegex(t *testing.T) {
158161
require.Contains(t, err.Error(), "no named capture groups")
159162
})
160163
}
164+
165+
func TestRegexParserConfig(t *testing.T) {
166+
expect := NewRegexParserConfig("test")
167+
expect.Regex = "test123"
168+
expect.ParseFrom = entry.NewRecordField("from")
169+
expect.ParseTo = entry.NewRecordField("to")
170+
171+
t.Run("mapstructure", func(t *testing.T) {
172+
input := map[string]interface{}{
173+
"id": "test",
174+
"type": "regex_parser",
175+
"regex": "test123",
176+
"parse_from": "$.from",
177+
"parse_to": "$.to",
178+
"on_error": "send",
179+
}
180+
var actual RegexParserConfig
181+
err := helper.UnmarshalMapstructure(input, &actual)
182+
require.NoError(t, err)
183+
require.Equal(t, expect, &actual)
184+
})
185+
186+
t.Run("yaml", func(t *testing.T) {
187+
input := `
188+
type: regex_parser
189+
id: test
190+
on_error: "send"
191+
regex: "test123"
192+
parse_from: $.from
193+
parse_to: $.to`
194+
var actual RegexParserConfig
195+
err := yaml.Unmarshal([]byte(input), &actual)
196+
require.NoError(t, err)
197+
require.Equal(t, expect, &actual)
198+
})
199+
}

‎operator/builtin/parser/severity/severity.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ func NewSeverityParserConfig(operatorID string) *SeverityParserConfig {
3636

3737
// SeverityParserConfig is the configuration of a severity parser operator.
3838
type SeverityParserConfig struct {
39-
helper.TransformerConfig `yaml:",inline"`
40-
helper.SeverityParserConfig `yaml:",omitempty,inline"`
39+
helper.TransformerConfig `mapstructure:",squash" yaml:",inline"`
40+
helper.SeverityParserConfig `mapstructure:",omitempty,squash" yaml:",omitempty,inline"`
4141
}
4242

4343
// Build will build a time parser operator.

‎operator/builtin/parser/severity/severity_test.go

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,14 @@ package severity
1717
import (
1818
"testing"
1919

20+
"github.com/stretchr/testify/mock"
21+
"github.com/stretchr/testify/require"
22+
"gopkg.in/yaml.v2"
23+
2024
"github.com/open-telemetry/opentelemetry-log-collection/entry"
2125
"github.com/open-telemetry/opentelemetry-log-collection/operator"
2226
"github.com/open-telemetry/opentelemetry-log-collection/operator/helper"
2327
"github.com/open-telemetry/opentelemetry-log-collection/testutil"
24-
"github.com/stretchr/testify/mock"
25-
"github.com/stretchr/testify/require"
2628
)
2729

2830
type severityTestCase struct {
@@ -360,3 +362,38 @@ func makeTestEntry(field entry.Field, value interface{}) *entry.Entry {
360362
e.Set(field, value)
361363
return e
362364
}
365+
366+
func TestSeverityParserConfig(t *testing.T) {
367+
expect := NewSeverityParserConfig("test")
368+
parseFrom := entry.NewRecordField("from")
369+
expect.ParseFrom = &parseFrom
370+
expect.Preset = "test"
371+
372+
t.Run("mapstructure", func(t *testing.T) {
373+
input := map[string]interface{}{
374+
"id": "test",
375+
"type": "severity_parser",
376+
"parse_from": "$.from",
377+
"on_error": "send",
378+
"preset": "test",
379+
}
380+
var actual SeverityParserConfig
381+
err := helper.UnmarshalMapstructure(input, &actual)
382+
require.NoError(t, err)
383+
require.Equal(t, expect, &actual)
384+
})
385+
386+
t.Run("yaml", func(t *testing.T) {
387+
input := `
388+
type: severity_parser
389+
id: test
390+
on_error: "send"
391+
parse_from: $.from
392+
preset: test
393+
`
394+
var actual SeverityParserConfig
395+
err := yaml.Unmarshal([]byte(input), &actual)
396+
require.NoError(t, err)
397+
require.Equal(t, expect, &actual)
398+
})
399+
}

‎operator/builtin/parser/syslog/syslog.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ func NewSyslogParserConfig(operatorID string) *SyslogParserConfig {
4040

4141
// SyslogParserConfig is the configuration of a syslog parser operator.
4242
type SyslogParserConfig struct {
43-
helper.ParserConfig `yaml:",inline"`
43+
helper.ParserConfig `mapstructure:",squash" yaml:",inline"`
4444

45-
Protocol string `json:"protocol,omitempty" yaml:"protocol,omitempty"`
46-
Location string `json:"location,omitempty" yaml:"location,omitempty"`
45+
Protocol string `mapstructure:"protocol,omitempty" json:"protocol,omitempty" yaml:"protocol,omitempty"`
46+
Location string `mapstructure:"location,omitempty" json:"location,omitempty" yaml:"location,omitempty"`
4747
}
4848

4949
// Build will build a JSON parser operator.

‎operator/builtin/parser/syslog/syslog_test.go

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,13 @@ import (
1919
"testing"
2020
"time"
2121

22+
"github.com/stretchr/testify/require"
23+
"gopkg.in/yaml.v2"
24+
2225
"github.com/open-telemetry/opentelemetry-log-collection/entry"
2326
"github.com/open-telemetry/opentelemetry-log-collection/operator"
27+
"github.com/open-telemetry/opentelemetry-log-collection/operator/helper"
2428
"github.com/open-telemetry/opentelemetry-log-collection/testutil"
25-
"github.com/stretchr/testify/require"
2629
)
2730

2831
func TestSyslogParser(t *testing.T) {
@@ -62,3 +65,39 @@ func TestSyslogParser(t *testing.T) {
6265
})
6366
}
6467
}
68+
69+
func TestSyslogParserConfig(t *testing.T) {
70+
expect := NewSyslogParserConfig("test")
71+
expect.Protocol = "rfc3164"
72+
expect.ParseFrom = entry.NewRecordField("from")
73+
expect.ParseTo = entry.NewRecordField("to")
74+
75+
t.Run("mapstructure", func(t *testing.T) {
76+
input := map[string]interface{}{
77+
"id": "test",
78+
"type": "syslog_parser",
79+
"protocol": "rfc3164",
80+
"parse_from": "$.from",
81+
"parse_to": "$.to",
82+
"on_error": "send",
83+
}
84+
var actual SyslogParserConfig
85+
err := helper.UnmarshalMapstructure(input, &actual)
86+
require.NoError(t, err)
87+
require.Equal(t, expect, &actual)
88+
})
89+
90+
t.Run("yaml", func(t *testing.T) {
91+
input := `
92+
type: syslog_parser
93+
id: test
94+
on_error: "send"
95+
protocol: "rfc3164"
96+
parse_from: $.from
97+
parse_to: $.to`
98+
var actual SyslogParserConfig
99+
err := yaml.Unmarshal([]byte(input), &actual)
100+
require.NoError(t, err)
101+
require.Equal(t, expect, &actual)
102+
})
103+
}

‎operator/builtin/parser/time/time.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ func NewTimeParserConfig(operatorID string) *TimeParserConfig {
3636

3737
// TimeParserConfig is the configuration of a time parser operator.
3838
type TimeParserConfig struct {
39-
helper.TransformerConfig `yaml:",inline"`
40-
helper.TimeParser `yaml:",omitempty,inline"`
39+
helper.TransformerConfig `mapstructure:",squash" yaml:",inline"`
40+
helper.TimeParser `mapstructure:",omitempty,squash" yaml:",omitempty,inline"`
4141
}
4242

4343
// Build will build a time parser operator.

‎operator/builtin/parser/time/time_test.go

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,14 @@ import (
1919
"testing"
2020
"time"
2121

22+
"github.com/stretchr/testify/mock"
23+
"github.com/stretchr/testify/require"
24+
"gopkg.in/yaml.v2"
25+
2226
"github.com/open-telemetry/opentelemetry-log-collection/entry"
2327
"github.com/open-telemetry/opentelemetry-log-collection/operator"
2428
"github.com/open-telemetry/opentelemetry-log-collection/operator/helper"
2529
"github.com/open-telemetry/opentelemetry-log-collection/testutil"
26-
"github.com/stretchr/testify/mock"
27-
"github.com/stretchr/testify/require"
2830
)
2931

3032
func TestIsZero(t *testing.T) {
@@ -442,3 +444,39 @@ func parseTimeTestConfig(layoutType, layout string, parseFrom entry.Field) *Time
442444
}
443445
return cfg
444446
}
447+
448+
func TestTimeParserConfig(t *testing.T) {
449+
expect := parseTimeTestConfig(helper.GotimeKey, "Mon Jan 2 15:04:05 MST 2006", entry.NewRecordField("from"))
450+
t.Run("mapstructure", func(t *testing.T) {
451+
input := map[string]interface{}{
452+
"id": "test_operator_id",
453+
"type": "time_parser",
454+
"output": []string{"output1"},
455+
"on_error": "send",
456+
"layout": "Mon Jan 2 15:04:05 MST 2006",
457+
"layout_type": "gotime",
458+
"parse_from": "$.from",
459+
}
460+
var actual TimeParserConfig
461+
err := helper.UnmarshalMapstructure(input, &actual)
462+
require.NoError(t, err)
463+
require.Equal(t, expect, &actual)
464+
})
465+
466+
t.Run("yaml", func(t *testing.T) {
467+
input := `
468+
type: time_parser
469+
id: test_operator_id
470+
on_error: "send"
471+
parse_from: $.from
472+
layout_type: gotime
473+
layout: "Mon Jan 2 15:04:05 MST 2006"
474+
output:
475+
- output1
476+
`
477+
var actual TimeParserConfig
478+
err := yaml.Unmarshal([]byte(input), &actual)
479+
require.NoError(t, err)
480+
require.Equal(t, expect, &actual)
481+
})
482+
}

‎operator/builtin/parser/uri/uri.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func NewURIParserConfig(operatorID string) *URIParserConfig {
3838

3939
// URIParserConfig is the configuration of a uri parser operator.
4040
type URIParserConfig struct {
41-
helper.ParserConfig `yaml:",inline"`
41+
helper.ParserConfig `mapstructure:",squash" yaml:",inline"`
4242
}
4343

4444
// Build will build a uri parser operator.

0 commit comments

Comments
 (0)