5
5
package io .airbyte .integrations .source .relationaldb ;
6
6
7
7
import static io .airbyte .integrations .base .errors .messages .ErrorMessage .getErrorMessage ;
8
+ import static io .airbyte .protocol .models .v0 .CatalogHelpers .fieldsToJsonSchema ;
8
9
9
10
import com .fasterxml .jackson .databind .JsonNode ;
10
11
import com .google .common .base .Preconditions ;
@@ -161,6 +162,8 @@ public AutoCloseableIterator<AirbyteMessage> read(final JsonNode config,
161
162
162
163
validateCursorFieldForIncrementalTables (fullyQualifiedTableNameToInfo , catalog , database );
163
164
165
+ logSourceSchemaChange (fullyQualifiedTableNameToInfo , catalog );
166
+
164
167
final List <AutoCloseableIterator <AirbyteMessage >> incrementalIterators =
165
168
getIncrementalIterators (database , catalog , fullyQualifiedTableNameToInfo , stateManager ,
166
169
emittedAt );
@@ -180,6 +183,36 @@ public AutoCloseableIterator<AirbyteMessage> read(final JsonNode config,
180
183
});
181
184
}
182
185
186
+ // in case of user manually modified source table schema but did not refresh it and save into the
187
+ // catalog - it can lead to sync failure. This method compare actual schema vs catalog schema
188
+ private void logSourceSchemaChange (Map <String , TableInfo <CommonField <DataType >>> fullyQualifiedTableNameToInfo ,
189
+ ConfiguredAirbyteCatalog catalog ) {
190
+ for (final ConfiguredAirbyteStream airbyteStream : catalog .getStreams ()) {
191
+ final AirbyteStream stream = airbyteStream .getStream ();
192
+ final String fullyQualifiedTableName = getFullyQualifiedTableName (stream .getNamespace (),
193
+ stream .getName ());
194
+ if (!fullyQualifiedTableNameToInfo .containsKey (fullyQualifiedTableName )) {
195
+ continue ;
196
+ }
197
+ final TableInfo <CommonField <DataType >> table = fullyQualifiedTableNameToInfo .get (fullyQualifiedTableName );
198
+ final List <Field > fields = table .getFields ()
199
+ .stream ()
200
+ .map (this ::toField )
201
+ .distinct ()
202
+ .collect (Collectors .toList ());
203
+ final JsonNode currentJsonSchema = fieldsToJsonSchema (fields );
204
+
205
+ final JsonNode catalogSchema = stream .getJsonSchema ();
206
+ if (!catalogSchema .equals (currentJsonSchema )) {
207
+ LOGGER .warn (
208
+ "Source schema changed for table {}! Actual schema: {}. Catalog schema: {}" ,
209
+ fullyQualifiedTableName ,
210
+ currentJsonSchema ,
211
+ catalogSchema );
212
+ }
213
+ }
214
+ }
215
+
183
216
private void validateCursorFieldForIncrementalTables (
184
217
final Map <String , TableInfo <CommonField <DataType >>> tableNameToTable ,
185
218
final ConfiguredAirbyteCatalog catalog ,
0 commit comments