Skip to content

Commit 0414536

Browse files
abuchanan-airbyteabuchanan-airbyte
authored
feat: json logs from the bootloader (#155)
1 parent 4dd803e commit 0414536

File tree

5 files changed

+90
-56
lines changed

5 files changed

+90
-56
lines changed

‎internal/cmd/local/helm/airbyte_values.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ func BuildAirbyteValues(ctx context.Context, opts ValuesOpts) (string, error) {
2525
"global.auth.enabled=true",
2626
"global.jobs.resources.limits.cpu=3",
2727
"global.jobs.resources.limits.memory=4Gi",
28+
"airbyte-bootloader.env_vars.PLATFORM_LOG_FORMAT=json",
2829
}
2930

3031
span.SetAttributes(

‎internal/cmd/local/local/install.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -442,10 +442,10 @@ func (c *Command) streamPodLogs(ctx context.Context, namespace, podName, prefix
442442

443443
s := newLogScanner(r)
444444
for s.Scan() {
445-
if s.line.level == "ERROR" {
446-
pterm.Error.Printfln("%s: %s", prefix, s.line.msg)
445+
if s.line.Level == "ERROR" {
446+
pterm.Error.Printfln("%s: %s", prefix, s.line.Message)
447447
} else {
448-
pterm.Debug.Printfln("%s: %s", prefix, s.line.msg)
448+
pterm.Debug.Printfln("%s: %s", prefix, s.line.Message)
449449
}
450450
}
451451

‎internal/cmd/local/local/log_utils.go

Lines changed: 75 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,10 @@ package local
22

33
import (
44
"bufio"
5+
"encoding/json"
56
"io"
6-
"regexp"
7-
"strings"
87
)
98

10-
// 2024-09-10 20:16:24 WARN i.m.s.r.u.Loggers$Slf4JLogger(warn):299 - [273....
11-
var logRx = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \x1b\[(?:1;)?\d+m(?P<level>[A-Z]+)\x1b\[m (?P<msg>\S+ - .*)`)
12-
13-
type logLine struct {
14-
msg string
15-
level string
16-
}
17-
189
type logScanner struct {
1910
scanner *bufio.Scanner
2011
line logLine
@@ -23,10 +14,6 @@ type logScanner struct {
2314
func newLogScanner(r io.Reader) *logScanner {
2415
return &logScanner{
2516
scanner: bufio.NewScanner(r),
26-
line: logLine{
27-
msg: "",
28-
level: "DEBUG",
29-
},
3017
}
3118
}
3219

@@ -36,26 +23,86 @@ func (j *logScanner) Scan() bool {
3623
return false
3724
}
3825

39-
// skip java stacktrace noise
40-
if strings.HasPrefix(j.scanner.Text(), "\tat ") || strings.HasPrefix(j.scanner.Text(), "\t... ") {
41-
continue
42-
}
43-
44-
m := logRx.FindSubmatch(j.scanner.Bytes())
45-
46-
if m != nil {
47-
j.line.msg = string(m[2])
48-
j.line.level = string(m[1])
26+
var data logLine
27+
err := json.Unmarshal(j.scanner.Bytes(), &data)
28+
// not all lines are JSON. don't propogate errors, just include the full line.
29+
if err != nil {
30+
j.line = logLine{Message: j.scanner.Text()}
4931
} else {
50-
// Some logs aren't from java (e.g. temporal) or they have a different format,
51-
// or the log covers multiple lines (e.g. java stack trace). In that case, use the full line
52-
// and reuse the level of the previous line.
53-
j.line.msg = j.scanner.Text()
32+
j.line = data
5433
}
34+
5535
return true
5636
}
5737
}
5838

5939
func (j *logScanner) Err() error {
6040
return j.scanner.Err()
6141
}
42+
43+
/*
44+
{
45+
"timestamp": 1734712334950,
46+
"message": "Unable to bootstrap Airbyte environment.",
47+
"level": "ERROR",
48+
"logSource": "platform",
49+
"caller": {
50+
"className": "io.airbyte.bootloader.Application",
51+
"methodName": "main",
52+
"lineNumber": 28,
53+
"threadName": "main"
54+
},
55+
"throwable": {
56+
"cause": {
57+
"cause": null,
58+
"stackTrace": [
59+
{
60+
"cn": "io.airbyte.bootloader.Application",
61+
"ln": 25,
62+
"mn": "main"
63+
}
64+
],
65+
"message": "Unable to connect to the database.",
66+
"suppressed": [],
67+
"localizedMessage": "Unable to connect to the database."
68+
},
69+
"stackTrace": [
70+
{
71+
"cn": "io.airbyte.bootloader.Application",
72+
"ln": 25,
73+
"mn": "main"
74+
}
75+
],
76+
"message": "Database availability check failed.",
77+
"suppressed": [],
78+
"localizedMessage": "Database availability check failed."
79+
}
80+
}
81+
*/
82+
type logLine struct {
83+
Timestamp int64 `json:"timestamp"`
84+
Message string `json:"message"`
85+
Level string `json:"level"`
86+
LogSource string `json:"logSource"`
87+
Caller *logCaller `json:"caller"`
88+
Throwable *logThrowable `json:throwable`
89+
}
90+
91+
type logCaller struct {
92+
ClassName string `json:"className"`
93+
MethodName string `json:"methodName"`
94+
LineNumber int `json:"lineNumber"`
95+
ThreadName string `json:"threadName"`
96+
}
97+
98+
type logStackElement struct {
99+
ClassName string `json:"cn"`
100+
LineNumber int `json:"ln"`
101+
MethodName string `json:"mn"`
102+
}
103+
104+
type logThrowable struct {
105+
Cause *logThrowable `json:"cause"`
106+
Stacktrace []logStackElement `json:"stackTrace"`
107+
Message string `json:"message"`
108+
}

‎internal/cmd/local/local/log_utils_test.go

Lines changed: 8 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,8 @@ import (
66
)
77

88
var testLogs = strings.TrimSpace(`
9-
2024-09-12 15:56:25 INFO i.a.d.c.DatabaseAvailabilityCheck(check):49 - Database is not ready yet. Please wait a moment, it might still be initializing...
10-
2024-09-12 15:56:30 WARN i.m.s.r.u.Loggers$Slf4JLogger(warn):299 - [54bd6014, L:/127.0.0.1:52991 - R:localhost/127.0.0.1:8125] An exception has been observed post termination, use DEBUG level to see the full stack: java.net.PortUnreachableException: recvAddress(..) failed: Connection refused
11-
2024-09-12 15:56:31 ERROR i.a.b.Application(main):25 - Unable to bootstrap Airbyte environment.
12-
io.airbyte.db.init.DatabaseInitializationException: Database availability check failed.
13-
at io.airbyte.db.init.DatabaseInitializer.initialize(DatabaseInitializer.java:54) ~[io.airbyte.airbyte-db-db-lib-0.64.3.jar:?]
14-
at io.airbyte.bootloader.Bootloader.initializeDatabases(Bootloader.java:229) ~[io.airbyte-airbyte-bootloader-0.64.3.jar:?]
15-
at io.airbyte.bootloader.Bootloader.load(Bootloader.java:104) ~[io.airbyte-airbyte-bootloader-0.64.3.jar:?]
16-
at io.airbyte.bootloader.Application.main(Application.java:22) [io.airbyte-airbyte-bootloader-0.64.3.jar:?]
17-
Caused by: io.airbyte.db.check.DatabaseCheckException: Unable to connect to the database.
18-
at io.airbyte.db.check.DatabaseAvailabilityCheck.check(DatabaseAvailabilityCheck.java:40) ~[io.airbyte.airbyte-db-db-lib-0.64.3.jar:?]
19-
at io.airbyte.db.init.DatabaseInitializer.initialize(DatabaseInitializer.java:45) ~[io.airbyte.airbyte-db-db-lib-0.64.3.jar:?]
20-
... 3 more
21-
2024-09-12 15:56:31 INFO i.m.r.Micronaut(lambda$start$0):118 - Embedded Application shutting down
22-
2024-09-12T15:56:33.125352208Z Thread-4 INFO Loading mask data from '/seed/specs_secrets_mask.yaml
9+
nonjsonline
10+
{"timestamp":1734723317023,"message":"Waiting for database to become available...","level":"WARN","logSource":"platform","caller":{"className":"io.airbyte.db.check.DatabaseAvailabilityCheck","methodName":"check","lineNumber":38,"threadName":"main"},"throwable":null}
2311
`)
2412

2513
func TestJavaLogScanner(t *testing.T) {
@@ -28,22 +16,17 @@ func TestJavaLogScanner(t *testing.T) {
2816
expectLogLine := func(level, msg string) {
2917
s.Scan()
3018

31-
if s.line.level != level {
32-
t.Errorf("expected level %q but got %q", level, s.line.level)
19+
if s.line.Level != level {
20+
t.Errorf("expected level %q but got %q", level, s.line.Level)
3321
}
34-
if s.line.msg != msg {
35-
t.Errorf("expected msg %q but got %q", msg, s.line.msg)
22+
if s.line.Message != msg {
23+
t.Errorf("expected msg %q but got %q", msg, s.line.Message)
3624
}
3725
if s.Err() != nil {
3826
t.Errorf("unexpected error %v", s.Err())
3927
}
4028
}
4129

42-
expectLogLine("INFO", "i.a.d.c.DatabaseAvailabilityCheck(check):49 - Database is not ready yet. Please wait a moment, it might still be initializing...")
43-
expectLogLine("WARN", "i.m.s.r.u.Loggers$Slf4JLogger(warn):299 - [54bd6014, L:/127.0.0.1:52991 - R:localhost/127.0.0.1:8125] An exception has been observed post termination, use DEBUG level to see the full stack: java.net.PortUnreachableException: recvAddress(..) failed: Connection refused")
44-
expectLogLine("ERROR", "i.a.b.Application(main):25 - Unable to bootstrap Airbyte environment.")
45-
expectLogLine("ERROR", "io.airbyte.db.init.DatabaseInitializationException: Database availability check failed.")
46-
expectLogLine("ERROR", "Caused by: io.airbyte.db.check.DatabaseCheckException: Unable to connect to the database.")
47-
expectLogLine("INFO", "i.m.r.Micronaut(lambda$start$0):118 - Embedded Application shutting down")
48-
expectLogLine("INFO", "2024-09-12T15:56:33.125352208Z Thread-4 INFO Loading mask data from '/seed/specs_secrets_mask.yaml")
30+
expectLogLine("", "nonjsonline")
31+
expectLogLine("WARN", "Waiting for database to become available...")
4932
}

‎internal/cmd/local/local/testdata/expected-default.values.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
airbyte-bootloader:
2+
env_vars:
3+
PLATFORM_LOG_FORMAT: json
14
global:
25
auth:
36
enabled: "true"

0 commit comments

Comments
 (0)