Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Draft] confgenerator : Add LoggingCompositeReceiver to standardize implementation of 3rd party app logging receivers. #1896

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft

[Draft] confgenerator : Add LoggingCompositeReceiver to standardize implementation of 3rd party app logging receivers. #1896

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cd0bfb7
Create generic composite logging receivers and use for 3p apps receiv…
franciscovalentecastro Feb 20, 2025
a556f67
Update 3p app receiver definition.
franciscovalentecastro Feb 21, 2025
4b28148
Update goldens.
franciscovalentecastro Feb 21, 2025
ddd00c0
Update iis_access implementation.
franciscovalentecastro Feb 21, 2025
eee0170
Use `LoggingProcessorMixin` for readability purposes.
franciscovalentecastro Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 12 additions & 14 deletions apps/apache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,12 +143,11 @@ func (LoggingProcessorApacheAccess) Type() string {
return "apache_access"
}

type LoggingReceiverApacheAccess struct {
LoggingProcessorApacheAccess `yaml:",inline"`
type LoggingReceiverApacheAccessMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverApacheAccess) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverApacheAccessMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
// Default log file path on Debian / Ubuntu
Expand All @@ -159,17 +158,14 @@ func (r LoggingReceiverApacheAccess) Components(ctx context.Context, tag string)
"/var/log/httpd/access_log",
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorApacheAccess.Components(ctx, tag, "apache_access")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

type LoggingReceiverApacheError struct {
LoggingProcessorApacheError `yaml:",inline"`
type LoggingReceiverApacheErrorMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverApacheError) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverApacheErrorMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
// Default log file path on Debian / Ubuntu
Expand All @@ -180,14 +176,16 @@ func (r LoggingReceiverApacheError) Components(ctx context.Context, tag string)
"/var/log/httpd/error_log",
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorApacheError.Components(ctx, tag, "apache_error")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorApacheAccess{} })
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorApacheError{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverApacheAccess{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverApacheError{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverApacheAccessMixin, LoggingProcessorApacheAccess]{}
})
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverApacheErrorMixin, LoggingProcessorApacheError]{}
})
}
39 changes: 18 additions & 21 deletions apps/cassandra.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,48 +234,41 @@ func (p LoggingProcessorCassandraGC) Components(ctx context.Context, tag string,
return c
}

type LoggingReceiverCassandraSystem struct {
LoggingProcessorCassandraSystem `yaml:",inline"`
type LoggingReceiverCassandraSystemMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverCassandraSystem) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverCassandraSystemMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
// Default log file path on Debian / Ubuntu / RHEL / CentOS
"/var/log/cassandra/system*.log",
// No default install position / log path for SLES
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorCassandraSystem.Components(ctx, tag, "cassandra_system")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

type LoggingReceiverCassandraDebug struct {
LoggingProcessorCassandraDebug `yaml:",inline"`
type LoggingReceiverCassandraDebugMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverCassandraDebug) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverCassandraDebugMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
// Default log file path on Debian / Ubuntu / RHEL / CentOS
"/var/log/cassandra/debug*.log",
// No default install position / log path for SLES
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorCassandraDebug.Components(ctx, tag, "cassandra_debug")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

type LoggingReceiverCassandraGC struct {
LoggingProcessorCassandraGC `yaml:",inline"`
type LoggingReceiverCassandraGCMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverCassandraGC) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverCassandraGCMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
// Default log file path on Debian / Ubuntu / RHEL / CentOS for JDK 8
Expand All @@ -285,16 +278,20 @@ func (r LoggingReceiverCassandraGC) Components(ctx context.Context, tag string)
// No default install position / log path for SLES
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorCassandraGC.Components(ctx, tag, "cassandra_gc")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorCassandraSystem{} })
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorCassandraDebug{} })
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorCassandraGC{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverCassandraSystem{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverCassandraDebug{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverCassandraGC{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverCassandraSystemMixin, LoggingProcessorCassandraSystem]{}
})
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverCassandraDebugMixin, LoggingProcessorCassandraDebug]{}
})
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverCassandraGCMixin, LoggingProcessorCassandraGC]{}
})
}
13 changes: 6 additions & 7 deletions apps/couchdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,11 @@ func (p LoggingProcessorCouchdb) Components(ctx context.Context, tag string, uid
return c
}

type LoggingReceiverCouchdb struct {
LoggingProcessorCouchdb `yaml:",inline"`
type LoggingReceiverCouchdbMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverCouchdb) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverCouchdbMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
// Default log file
Expand All @@ -178,12 +177,12 @@ func (r LoggingReceiverCouchdb) Components(ctx context.Context, tag string) []fl
},
}

c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorCouchdb.Components(ctx, tag, "couchdb")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorCouchdb{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverCouchdb{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverCouchdbMixin, LoggingProcessorCouchdb]{}
})
}
24 changes: 12 additions & 12 deletions apps/elasticsearch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,12 +268,11 @@ func (p LoggingProcessorElasticsearchGC) Components(ctx context.Context, tag, ui
return c
}

type LoggingReceiverElasticsearchJson struct {
LoggingProcessorElasticsearchJson `yaml:",inline"`
type LoggingReceiverElasticsearchJsonMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline"`
}

func (r LoggingReceiverElasticsearchJson) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverElasticsearchJsonMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
// Default JSON logs for Elasticsearch
r.IncludePaths = []string{
Expand Down Expand Up @@ -307,28 +306,29 @@ func (r LoggingReceiverElasticsearchJson) Components(ctx context.Context, tag st
},
}

c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
return append(c, r.LoggingProcessorElasticsearchJson.Components(ctx, tag, "elasticsearch_json")...)
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

type LoggingReceiverElasticsearchGC struct {
LoggingProcessorElasticsearchGC `yaml:",inline"`
type LoggingReceiverElasticsearchGCMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline"`
}

func (r LoggingReceiverElasticsearchGC) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverElasticsearchGCMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
// Default GC log for Elasticsearch
r.IncludePaths = []string{
"/var/log/elasticsearch/gc.log",
}
}

c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
return append(c, r.LoggingProcessorElasticsearchGC.Components(ctx, tag, "elasticsearch_gc")...)
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchJson{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverElasticsearchGC{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverElasticsearchJsonMixin, LoggingProcessorElasticsearchJson]{}
})
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverElasticsearchGCMixin, LoggingProcessorElasticsearchGC]{}
})
}
13 changes: 6 additions & 7 deletions apps/flink.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,25 +141,24 @@ func (p LoggingProcessorFlink) Components(ctx context.Context, tag string, uid s
return c
}

type LoggingReceiverFlink struct {
LoggingProcessorFlink `yaml:",inline"`
type LoggingReceiverFlinkMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverFlink) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverFlinkMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
"/opt/flink/log/flink-*-standalonesession-*.log",
"/opt/flink/log/flink-*-taskexecutor-*.log",
"/opt/flink/log/flink-*-client-*.log",
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorFlink.Components(ctx, tag, "flink")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverFlinkMixin, LoggingProcessorFlink]{}
})
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorFlink{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverFlink{} })
}
13 changes: 6 additions & 7 deletions apps/hadoop.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,11 @@ func (p LoggingProcessorHadoop) Components(ctx context.Context, tag, uid string)
return c
}

type LoggingReceiverHadoop struct {
LoggingProcessorHadoop `yaml:",inline"`
type LoggingReceiverHadoopMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline"`
}

func (r LoggingReceiverHadoop) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverHadoopMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
// Default logs for hadoop
r.IncludePaths = []string{
Expand All @@ -132,11 +131,11 @@ func (r LoggingReceiverHadoop) Components(ctx context.Context, tag string) []flu
},
}

c := r.LoggingReceiverFilesMixin.Components(ctx, tag)

return append(c, r.LoggingProcessorHadoop.Components(ctx, tag, "hadoop")...)
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverHadoop{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverHadoopMixin, LoggingProcessorHadoop]{}
})
}
12 changes: 6 additions & 6 deletions apps/hbase.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,24 +125,24 @@ func (p LoggingProcessorHbaseSystem) Components(ctx context.Context, tag string,
return c
}

type SystemLoggingReceiverHbase struct {
type SystemLoggingReceiverHbaseMixin struct {
LoggingProcessorHbaseSystem `yaml:",inline"`
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r SystemLoggingReceiverHbase) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r SystemLoggingReceiverHbaseMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
"/opt/hbase/logs/hbase-*-regionserver-*.log",
"/opt/hbase/logs/hbase-*-master-*.log",
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorHbaseSystem.Components(ctx, tag, "hbase_system")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorHbaseSystem{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &SystemLoggingReceiverHbase{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[SystemLoggingReceiverHbaseMixin, LoggingProcessorHbaseSystem]{}
})
}
17 changes: 8 additions & 9 deletions apps/iis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ type LoggingProcessorIisAccess struct {
confgenerator.ConfigComponent `yaml:",inline"`
}

func (*LoggingProcessorIisAccess) Type() string {
func (LoggingProcessorIisAccess) Type() string {
return "iis_access"
}

Expand Down Expand Up @@ -184,7 +184,7 @@ const (
`
)

func (p *LoggingProcessorIisAccess) Components(ctx context.Context, tag, uid string) []fluentbit.Component {
func (p LoggingProcessorIisAccess) Components(ctx context.Context, tag, uid string) []fluentbit.Component {
c := confgenerator.LoggingProcessorParseRegex{
// Documentation:
// https://docs.microsoft.com/en-us/windows/win32/http/w3c-logging
Expand Down Expand Up @@ -248,23 +248,22 @@ func (p *LoggingProcessorIisAccess) Components(ctx context.Context, tag, uid str
return c
}

type LoggingReceiverIisAccess struct {
LoggingProcessorIisAccess `yaml:",inline"`
type LoggingReceiverIisAccessMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverIisAccess) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverIisAccessMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
`C:\inetpub\logs\LogFiles\W3SVC1\u_ex*`,
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorIisAccess.Components(ctx, tag, "iis_access")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverIisAccess{} }, platform.Windows)
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverIisAccessMixin, LoggingProcessorIisAccess]{}
}, platform.Windows)
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorIisAccess{} }, platform.Windows)
}
13 changes: 6 additions & 7 deletions apps/jetty.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,23 +71,22 @@ func (LoggingProcessorJettyAccess) Type() string {
return "jetty_access"
}

type LoggingReceiverJettyAccess struct {
LoggingProcessorJettyAccess `yaml:",inline"`
type LoggingReceiverJettyAccessMixin struct {
confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"`
}

func (r LoggingReceiverJettyAccess) Components(ctx context.Context, tag string) []fluentbit.Component {
func (r LoggingReceiverJettyAccessMixin) Components(ctx context.Context, tag string) []fluentbit.Component {
if len(r.IncludePaths) == 0 {
r.IncludePaths = []string{
"/opt/logs/*.request.log",
}
}
c := r.LoggingReceiverFilesMixin.Components(ctx, tag)
c = append(c, r.LoggingProcessorJettyAccess.Components(ctx, tag, "jetty_access")...)
return c
return r.LoggingReceiverFilesMixin.Components(ctx, tag)
}

func init() {
confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &LoggingProcessorJettyAccess{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverJettyAccess{} })
confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver {
return &confgenerator.LoggingCompositeReceiver[LoggingReceiverJettyAccessMixin, LoggingProcessorJettyAccess]{}
})
}
Loading
Loading