66
77import io .airbyte .integrations .destination .ExtendedNameTransformer ;
88import io .airbyte .integrations .standardtest .destination .comparator .AdvancedTestDataComparator ;
9+ import java .time .LocalDate ;
10+ import java .time .LocalDateTime ;
11+ import java .time .ZoneOffset ;
12+ import java .time .ZonedDateTime ;
13+ import java .time .format .DateTimeFormatter ;
14+ import java .time .format .DateTimeParseException ;
915import java .util .ArrayList ;
1016import java .util .List ;
17+ import org .slf4j .Logger ;
18+ import org .slf4j .LoggerFactory ;
1119
1220public class ClickhouseTestDataComparator extends AdvancedTestDataComparator {
1321
22+ private static final Logger LOGGER = LoggerFactory .getLogger (ClickhouseTestDataComparator .class );
1423 private final ExtendedNameTransformer namingResolver = new ExtendedNameTransformer ();
1524
25+ private static final String CLICKHOUSE_DATETIME_WITH_TZ_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSX" ;
26+
27+ // https://clickhouse.com/docs/en/sql-reference/data-types/date32/
28+ private final LocalDate minSupportedDate = LocalDate .parse ("1970-01-01" );
29+ private final LocalDate maxSupportedDate = LocalDate .parse ("2149-06-06" );
30+ private final ZonedDateTime minSupportedDateTime = ZonedDateTime .parse (
31+ "1925-01-01T00:00:00.000Z" );
32+ private final ZonedDateTime maxSupportedDateTime = ZonedDateTime .parse (
33+ "2283-11-10T20:23:45.000Z" );
34+
1635 @ Override
1736 protected List <String > resolveIdentifier (final String identifier ) {
1837 final List <String > result = new ArrayList <>();
@@ -26,4 +45,103 @@ protected List<String> resolveIdentifier(final String identifier) {
2645 return result ;
2746 }
2847
48+ @ Override
49+ protected boolean compareNumericValues (final String firstNumericValue ,
50+ final String secondNumericValue ) {
51+ // clickhouse stores double 1.14 as 1.1400000000000001
52+ // https://clickhouse.com/docs/en/sql-reference/data-types/float/
53+ double epsilon = 0.000000000000001d ;
54+
55+ double firstValue = Double .parseDouble (firstNumericValue );
56+ double secondValue = Double .parseDouble (secondNumericValue );
57+
58+ return Math .abs (firstValue - secondValue ) < epsilon ;
59+ }
60+
61+ @ Override
62+ protected boolean compareBooleanValues (final String firstValue , final String secondValue ) {
63+ return parseBool (firstValue ) == parseBool (secondValue );
64+ }
65+
66+ @ Override
67+ protected boolean compareDateValues (final String airbyteMessageValue ,
68+ final String destinationValue ) {
69+ final LocalDate expectedDate = LocalDate .parse (airbyteMessageValue );
70+ final LocalDate actualDate = LocalDate .parse (destinationValue );
71+
72+ if (expectedDate .isBefore (minSupportedDate ) || expectedDate .isAfter (maxSupportedDate )) {
73+ // inserting any dates that are out of supported range causes registers overflow in clickhouseDB,
74+ // so actually you end up with unpredicted values, more
75+ // https://clickhouse.com/docs/en/sql-reference/data-types/date32
76+ LOGGER .warn (
77+ "Test value is out of range and would be corrupted by Snowflake, so we skip this verification" );
78+ return true ;
79+ }
80+
81+ return actualDate .equals (expectedDate );
82+ }
83+
84+ @ Override
85+ protected boolean compareDateTimeWithTzValues (final String airbyteMessageValue ,
86+ final String destinationValue ) {
87+ try {
88+ ZonedDateTime airbyteDate = ZonedDateTime .parse (airbyteMessageValue ,
89+ getAirbyteDateTimeWithTzFormatter ()).withZoneSameInstant (ZoneOffset .UTC );
90+ ZonedDateTime destinationDate = parseDestinationDateWithTz (destinationValue );
91+
92+ if (airbyteDate .isBefore (minSupportedDateTime ) || airbyteDate .isAfter (maxSupportedDateTime )) {
93+ // inserting any dates that are out of supported range causes registers overflow in clickhouseDB,
94+ // so actually you end up with unpredicted values, more
95+ // https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
96+ LOGGER .warn (
97+ "Test value is out of range and would be corrupted by Snowflake, so we skip this verification" );
98+ return true ;
99+ }
100+ return airbyteDate .equals (destinationDate );
101+ } catch (DateTimeParseException e ) {
102+ LOGGER .warn (
103+ "Fail to convert values to ZonedDateTime. Try to compare as text. Airbyte value({}), Destination value ({}). Exception: {}" ,
104+ airbyteMessageValue , destinationValue , e );
105+ return compareTextValues (airbyteMessageValue , destinationValue );
106+ }
107+ }
108+
109+ @ Override
110+ protected ZonedDateTime parseDestinationDateWithTz (final String destinationValue ) {
111+ return ZonedDateTime .parse (destinationValue ,
112+ DateTimeFormatter .ofPattern (CLICKHOUSE_DATETIME_WITH_TZ_FORMAT )).withZoneSameInstant (
113+ ZoneOffset .UTC );
114+ }
115+
116+ @ Override
117+ protected boolean compareDateTimeValues (final String airbyteMessageValue ,
118+ final String destinationValue ) {
119+ final LocalDateTime expectedDateTime = LocalDateTime .parse (airbyteMessageValue );
120+ final LocalDateTime actualDateTime = LocalDateTime .parse (destinationValue ,
121+ DateTimeFormatter .ofPattern (CLICKHOUSE_DATETIME_WITH_TZ_FORMAT ));
122+
123+ if (expectedDateTime .isBefore (minSupportedDateTime .toLocalDateTime ())
124+ || expectedDateTime .isAfter (maxSupportedDateTime .toLocalDateTime ())) {
125+ // inserting any dates that are out of supported range causes registers overflow in clickhouseDB,
126+ // so actually you end up with unpredicted values, more
127+ // https://clickhouse.com/docs/en/sql-reference/data-types/datetime64
128+ LOGGER .warn (
129+ "Test value is out of range and would be corrupted by Snowflake, so we skip this verification" );
130+ return true ;
131+ }
132+
133+ return expectedDateTime .equals (actualDateTime );
134+ }
135+
136+ private boolean parseBool (final String valueAsString ) {
137+ // boolen as a String may be returned as true\false and as 0\1
138+ // https://clickhouse.com/docs/en/sql-reference/data-types/boolean
139+ try {
140+ return Integer .parseInt (valueAsString ) > 0 ;
141+ } catch (final NumberFormatException ex ) {
142+ return Boolean .parseBoolean (valueAsString );
143+ }
144+
145+ }
146+
29147}
0 commit comments