Skip to content

Commit 7cd0a70

Browse files
stephane-airbytestephane-airbyte
committed
write logs to file in addition to stdout when running java connector tests
1 parent c07915b commit 7cd0a70

File tree

3 files changed

+41
-9
lines changed

3 files changed

+41
-9
lines changed

‎airbyte-cdk/java/airbyte-cdk/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ MavenLocal debugging steps:
166166

167167
| Version | Date | Pull Request | Subject |
168168
|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------|
169+
| 0.20.7 | 2024-02-13 | [\#35236](https://github.com/airbytehq/airbyte/pull/35236) | output logs to files in addition to stdout when running tests |
169170
| 0.20.6 | 2024-02-12 | [\#35036](https://github.com/airbytehq/airbyte/pull/35036) | Add trace utility to emit analytics messages. |
170171
| 0.20.5 | 2024-02-13 | [\#34869](https://github.com/airbytehq/airbyte/pull/34869) | Don't emit final state in SourceStateIterator there is an underlying stream failure. |
171172
| 0.20.4 | 2024-02-12 | [\#35042](https://github.com/airbytehq/airbyte/pull/35042) | Use delegate's isDestinationV2 invocation in SshWrappedDestination. |

‎airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=0.20.6
1+
version=0.20.7

‎airbyte-cdk/java/airbyte-cdk/dependencies/src/main/resources/log4j2-test.xml

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,64 @@
22
<Configuration status="INFO">
33
<Properties>
44
<!-- Set the LOG_SCRUB_PATTERN env var to a regex pattern to scrub the log messages of secrets or any other unwanted content. -->
5-
<Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
5+
<Property name="jvm-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%highlight{%p}`%C{1.}(%M):%L - %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
66
<!--Logs the timestamp and log_source/application name in the beginning of the line if it exists with a > separator, and then always the rest of the line.-->
7-
<Property name="simple-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0}%replace{ %X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
7+
<Property name="container-log-pattern">%d{yyyy-MM-dd'T'HH:mm:ss,SSS}{GMT+0}`%replace{ %X{log_source}}{^ -}{} > %replace{%m}{$${env:LOG_SCRUB_PATTERN:-\*\*\*\*\*}}{*****}%n</Property>
88
<!-- Always log INFO by default. -->
99
<Property name="log-level">${sys:LOG_LEVEL:-${env:LOG_LEVEL:-INFO}}</Property>
10+
<Property name="logSubDir">${env:AIRBYTE_LOG_SUBDIR:-${date:yyyy-MM-dd'T'HH:mm:ss}}</Property>
11+
<Property name="logDir">build/test-logs/${logSubDir}</Property>
1012
</Properties>
1113

1214
<Appenders>
13-
<Console name="Default" target="SYSTEM_OUT">
15+
<Console name="JvmLogsStdOut" target="SYSTEM_OUT">
1416
<ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
1517
<KeyValuePair key="simple" value="true"/>
1618
</ContextMapFilter>
17-
<PatternLayout pattern="${default-pattern}"/>
19+
<PatternLayout pattern="${jvm-log-pattern}"/>
1820
</Console>
1921

20-
<Console name="SimpleDefault" target="SYSTEM_OUT">
22+
<Console name="ContainerLogsStdOut" target="SYSTEM_OUT">
2123
<ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
2224
<KeyValuePair key="simple" value="true"/>
2325
</ContextMapFilter>
24-
<PatternLayout pattern="${simple-pattern}"/>
26+
<PatternLayout pattern="${container-log-pattern}"/>
2527
</Console>
28+
29+
<File name="JvmLogsFile" fileName="${logDir}/airbyte_jvm.log">
30+
<ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
31+
<KeyValuePair key="simple" value="true"/>
32+
</ContextMapFilter>
33+
<PatternLayout pattern="${jvm-log-pattern}"/>
34+
</File>
35+
<File name="ContainerLogFiles" fileName="${logDir}/airbyte_containers.log">
36+
<ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
37+
<KeyValuePair key="simple" value="true"/>
38+
</ContextMapFilter>
39+
<PatternLayout pattern="${container-log-pattern}"/>
40+
</File>
41+
<File name="UnifiedFile-JvmLogs" fileName="${logDir}/airbyte_unified.log">
42+
<ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
43+
<KeyValuePair key="simple" value="true"/>
44+
</ContextMapFilter>
45+
<PatternLayout pattern="${jvm-log-pattern}"/>
46+
</File>
47+
<File name="UnifiedFile-ContainerLogs" fileName="${logDir}/airbyte_unified.log">
48+
<ContextMapFilter onMatch="ACCEPT" onMismatch="DENY">
49+
<KeyValuePair key="simple" value="true"/>
50+
</ContextMapFilter>
51+
<PatternLayout pattern="${container-log-pattern}"/>
52+
</File>
2653
</Appenders>
2754

2855
<Loggers>
2956
<Root level="${log-level}">
30-
<AppenderRef ref="Default"/>
31-
<AppenderRef ref="SimpleDefault"/>
57+
<AppenderRef ref="JvmLogsStdOut"/>
58+
<AppenderRef ref="ContainerLogsStdOut"/>
59+
<AppenderRef ref="JvmLogsFile"/>
60+
<AppenderRef ref="ContainerLogFiles"/>
61+
<AppenderRef ref="UnifiedFile-JvmLogs"/>
62+
<AppenderRef ref="UnifiedFile-ContainerLogs"/>
3263
</Root>
3364

3465
<Logger name="org.eclipse.jetty" level="INFO" />

0 commit comments

Comments
 (0)