4
4
5
5
package io .airbyte .integrations .debezium .internals ;
6
6
7
+ import static io .airbyte .db .DataTypeUtils .TIMETZ_FORMATTER ;
8
+ import static io .airbyte .db .jdbc .DateTimeConverter .convertToDate ;
9
+ import static io .airbyte .db .jdbc .DateTimeConverter .convertToTime ;
10
+ import static io .airbyte .db .jdbc .DateTimeConverter .convertToTimestamp ;
11
+ import static io .airbyte .db .jdbc .DateTimeConverter .convertToTimestampWithTimezone ;
12
+ import static org .apache .kafka .connect .data .Schema .OPTIONAL_BOOLEAN_SCHEMA ;
13
+ import static org .apache .kafka .connect .data .Schema .OPTIONAL_FLOAT64_SCHEMA ;
14
+ import static org .apache .kafka .connect .data .Schema .OPTIONAL_STRING_SCHEMA ;
15
+
7
16
import io .airbyte .db .jdbc .DateTimeConverter ;
8
17
import io .debezium .spi .converter .CustomConverter ;
9
18
import io .debezium .spi .converter .RelationalColumn ;
10
19
import io .debezium .time .Conversions ;
11
20
import java .math .BigDecimal ;
12
21
import java .nio .charset .StandardCharsets ;
22
+ import java .sql .SQLException ;
13
23
import java .time .LocalDate ;
14
24
import java .time .LocalTime ;
25
+ import java .time .OffsetTime ;
26
+ import java .time .format .DateTimeFormatter ;
27
+ import java .util .ArrayList ;
15
28
import java .util .Arrays ;
29
+ import java .util .Base64 ;
30
+ import java .util .List ;
16
31
import java .util .Locale ;
32
+ import java .util .Objects ;
17
33
import java .util .Properties ;
18
34
import java .util .concurrent .TimeUnit ;
35
+ import java .util .stream .Collectors ;
19
36
import org .apache .commons .codec .binary .Hex ;
20
37
import org .apache .kafka .connect .data .SchemaBuilder ;
38
+ import org .postgresql .jdbc .PgArray ;
21
39
import org .postgresql .util .PGInterval ;
22
40
import org .slf4j .Logger ;
23
41
import org .slf4j .LoggerFactory ;
@@ -33,6 +51,7 @@ public class PostgresConverter implements CustomConverter<SchemaBuilder, Relatio
33
51
private final String [] TEXT_TYPES =
34
52
{"VARCHAR" , "VARBINARY" , "BLOB" , "TEXT" , "LONGTEXT" , "TINYTEXT" , "MEDIUMTEXT" , "INVENTORY_ITEM" , "TSVECTOR" , "TSQUERY" , "PG_LSN" };
35
53
private final String [] NUMERIC_TYPES = {"NUMERIC" , "DECIMAL" };
54
+ private final String [] ARRAY_TYPES = {"_NAME" , "_NUMERIC" , "_BYTEA" , "_MONEY" , "_BIT" , "_DATE" , "_TIME" , "_TIMETZ" , "_TIMESTAMP" , "_TIMESTAMPTZ" };
36
55
private final String BYTEA_TYPE = "BYTEA" ;
37
56
38
57
@ Override
@@ -52,9 +71,22 @@ public void converterFor(final RelationalColumn field, final ConverterRegistrati
52
71
registerBytea (field , registration );
53
72
} else if (Arrays .stream (NUMERIC_TYPES ).anyMatch (s -> s .equalsIgnoreCase (field .typeName ()))) {
54
73
registerNumber (field , registration );
74
+ } else if (Arrays .stream (ARRAY_TYPES ).anyMatch (s -> s .equalsIgnoreCase (field .typeName ()))) {
75
+ registerArray (field , registration );
55
76
}
56
77
}
57
78
79
+ private void registerArray (RelationalColumn field , ConverterRegistration <SchemaBuilder > registration ) {
80
+ final String fieldType = field .typeName ().toUpperCase ();
81
+ final SchemaBuilder arraySchema = switch (fieldType ) {
82
+ case "_NUMERIC" , "_MONEY" -> SchemaBuilder .array (OPTIONAL_FLOAT64_SCHEMA );
83
+ case "_NAME" , "_DATE" , "_TIME" , "_TIMESTAMP" , "_TIMESTAMPTZ" , "_TIMETZ" , "_BYTEA" -> SchemaBuilder .array (OPTIONAL_STRING_SCHEMA );
84
+ case "_BIT" -> SchemaBuilder .array (OPTIONAL_BOOLEAN_SCHEMA );
85
+ default -> SchemaBuilder .array (OPTIONAL_STRING_SCHEMA );
86
+ };
87
+ registration .register (arraySchema , x -> convertArray (x , field ));
88
+ }
89
+
58
90
private void registerNumber (final RelationalColumn field , final ConverterRegistration <SchemaBuilder > registration ) {
59
91
registration .register (SchemaBuilder .string ().optional (), x -> {
60
92
if (x == null ) {
@@ -106,6 +138,72 @@ private void registerText(final RelationalColumn field, final ConverterRegistrat
106
138
});
107
139
}
108
140
141
+ private Object convertArray (Object x , RelationalColumn field ) {
142
+ final String fieldType = field .typeName ().toUpperCase ();
143
+ Object [] values = new Object [0 ];
144
+ try {
145
+ values = (Object []) ((PgArray ) x ).getArray ();
146
+ } catch (SQLException e ) {
147
+ LOGGER .error ("Failed to convert PgArray:" + e );
148
+ }
149
+ switch (fieldType ) {
150
+ // debezium currently cannot handle MONEY[] datatype and it's not implemented
151
+ case "_MONEY" :
152
+ // PgArray.getArray() trying to convert to Double instead of PgMoney
153
+ // due to incorrect type mapping in the postgres driver
154
+ // https://github.com/pgjdbc/pgjdbc/blob/d5ed52ef391670e83ae5265af2f7301c615ce4ca/pgjdbc/src/main/java/org/postgresql/jdbc/TypeInfoCache.java#L88
155
+ // and throws an exception, so a custom implementation of converting to String is used to get the
156
+ // value as is
157
+ final String nativeMoneyValue = ((PgArray ) x ).toString ();
158
+ final String substringM = Objects .requireNonNull (nativeMoneyValue ).substring (1 , nativeMoneyValue .length () - 1 );
159
+ final char currency = substringM .charAt (0 );
160
+ final String regex = "\\ " + currency ;
161
+ final List <String > myListM = new ArrayList <>(Arrays .asList (substringM .split (regex )));
162
+ return myListM .stream ()
163
+ // since the separator is the currency sign, all extra characters must be removed except for numbers
164
+ // and dots
165
+ .map (val -> val .replaceAll ("[^\\ d.]" , "" ))
166
+ .filter (money -> !money .isEmpty ())
167
+ .map (Double ::valueOf )
168
+ .collect (Collectors .toList ());
169
+ case "_NUMERIC" :
170
+ return Arrays .stream (values ).map (value -> value == null ? null : Double .valueOf (value .toString ())).collect (Collectors .toList ());
171
+ case "_TIME" :
172
+ return Arrays .stream (values ).map (value -> value == null ? null : convertToTime (value )).collect (Collectors .toList ());
173
+ case "_DATE" :
174
+ return Arrays .stream (values ).map (value -> value == null ? null : convertToDate (value )).collect (Collectors .toList ());
175
+ case "_TIMESTAMP" :
176
+ return Arrays .stream (values ).map (value -> value == null ? null : convertToTimestamp (value )).collect (Collectors .toList ());
177
+ case "_TIMESTAMPTZ" :
178
+ return Arrays .stream (values ).map (value -> value == null ? null : convertToTimestampWithTimezone (value )).collect (Collectors .toList ());
179
+ case "_TIMETZ" :
180
+
181
+ final List <String > timetzArr = new ArrayList <>();
182
+ final String nativeValue = ((PgArray ) x ).toString ();
183
+ final String substring = Objects .requireNonNull (nativeValue ).substring (1 , nativeValue .length () - 1 );
184
+ final List <String > times = new ArrayList <>(Arrays .asList (substring .split ("," )));
185
+ final DateTimeFormatter format = DateTimeFormatter .ofPattern ("HH:mm:ss[.SSSSSS]X" );
186
+
187
+ times .forEach (s -> {
188
+ if (s .equalsIgnoreCase ("NULL" )) {
189
+ timetzArr .add (null );
190
+ } else {
191
+ final OffsetTime parsed = OffsetTime .parse (s , format );
192
+ timetzArr .add (parsed .format (TIMETZ_FORMATTER ));
193
+ }
194
+ });
195
+ return timetzArr ;
196
+ case "_BYTEA" :
197
+ return Arrays .stream (values ).map (value -> Base64 .getEncoder ().encodeToString ((byte []) value )).collect (Collectors .toList ());
198
+ case "_BIT" :
199
+ return Arrays .stream (values ).map (value -> (Boolean ) value ).collect (Collectors .toList ());
200
+ case "_NAME" :
201
+ return Arrays .stream (values ).map (value -> (String ) value ).collect (Collectors .toList ());
202
+ default :
203
+ return new ArrayList <>();
204
+ }
205
+ }
206
+
109
207
private int getTimePrecision (final RelationalColumn field ) {
110
208
return field .scale ().orElse (-1 );
111
209
}
@@ -127,30 +225,20 @@ private void registerDate(final RelationalColumn field, final ConverterRegistrat
127
225
case "TIMESTAMP" :
128
226
if (x instanceof final Long l ) {
129
227
if (getTimePrecision (field ) <= 3 ) {
130
- return DateTimeConverter . convertToTimestamp (Conversions .toInstantFromMillis (l ));
228
+ return convertToTimestamp (Conversions .toInstantFromMillis (l ));
131
229
}
132
230
if (getTimePrecision (field ) <= 6 ) {
133
- return DateTimeConverter . convertToTimestamp (Conversions .toInstantFromMicros (l ));
231
+ return convertToTimestamp (Conversions .toInstantFromMicros (l ));
134
232
}
135
233
}
136
- return DateTimeConverter . convertToTimestamp (x );
234
+ return convertToTimestamp (x );
137
235
case "DATE" :
138
236
if (x instanceof Integer ) {
139
- return DateTimeConverter . convertToDate (LocalDate .ofEpochDay ((Integer ) x ));
237
+ return convertToDate (LocalDate .ofEpochDay ((Integer ) x ));
140
238
}
141
- return DateTimeConverter . convertToDate (x );
239
+ return convertToDate (x );
142
240
case "TIME" :
143
- if (x instanceof Long ) {
144
- if (getTimePrecision (field ) <= 3 ) {
145
- long l = Math .multiplyExact ((Long ) x , TimeUnit .MILLISECONDS .toNanos (1 ));
146
- return DateTimeConverter .convertToTime (LocalTime .ofNanoOfDay (l ));
147
- }
148
- if (getTimePrecision (field ) <= 6 ) {
149
- long l = Math .multiplyExact ((Long ) x , TimeUnit .MICROSECONDS .toNanos (1 ));
150
- return DateTimeConverter .convertToTime (LocalTime .ofNanoOfDay (l ));
151
- }
152
- }
153
- return DateTimeConverter .convertToTime (x );
241
+ return resolveTime (field , x );
154
242
case "INTERVAL" :
155
243
return convertInterval ((PGInterval ) x );
156
244
default :
@@ -159,6 +247,20 @@ private void registerDate(final RelationalColumn field, final ConverterRegistrat
159
247
});
160
248
}
161
249
250
+ private String resolveTime (RelationalColumn field , Object x ) {
251
+ if (x instanceof Long ) {
252
+ if (getTimePrecision (field ) <= 3 ) {
253
+ long l = Math .multiplyExact ((Long ) x , TimeUnit .MILLISECONDS .toNanos (1 ));
254
+ return DateTimeConverter .convertToTime (LocalTime .ofNanoOfDay (l ));
255
+ }
256
+ if (getTimePrecision (field ) <= 6 ) {
257
+ long l = Math .multiplyExact ((Long ) x , TimeUnit .MICROSECONDS .toNanos (1 ));
258
+ return DateTimeConverter .convertToTime (LocalTime .ofNanoOfDay (l ));
259
+ }
260
+ }
261
+ return DateTimeConverter .convertToTime (x );
262
+ }
263
+
162
264
private String convertInterval (final PGInterval pgInterval ) {
163
265
final StringBuilder resultInterval = new StringBuilder ();
164
266
formatDateUnit (resultInterval , pgInterval .getYears (), " year " );
0 commit comments