Skip to content

Commit ea1228d

Browse files
akashkulkoctavia-squidington-iii
authored and
marcosmarxm
committed
Revert "Better formatting/logging for pre-sync data" (airbytehq#25560)
* Revert "Better formatting/logging for pre-sync data (airbytehq#25459)" This reverts commit 0d9b7b4. * Fix dockerfile + docs * auto-bump connector version --------- Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 1efd354 commit ea1228d

File tree

10 files changed

+26
-69
lines changed

10 files changed

+26
-69
lines changed

‎airbyte-config-oss/init-oss/src/main/resources/seed/oss_catalog.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -21221,7 +21221,7 @@
2122121221
"sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",
2122221222
"name": "Postgres",
2122321223
"dockerRepository": "airbyte/source-postgres",
21224-
"dockerImageTag": "2.0.25",
21224+
"dockerImageTag": "2.0.26",
2122521225
"documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres",
2122621226
"icon": "postgresql.svg",
2122721227
"sourceType": "database",

‎airbyte-config-oss/init-oss/src/main/resources/seed/source_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1619,7 +1619,7 @@
16191619
- name: Postgres
16201620
sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
16211621
dockerRepository: airbyte/source-postgres
1622-
dockerImageTag: 2.0.25
1622+
dockerImageTag: 2.0.26
16231623
documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
16241624
icon: postgresql.svg
16251625
sourceType: database

‎airbyte-config-oss/init-oss/src/main/resources/seed/source_specs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12154,7 +12154,7 @@
1215412154
supportsNormalization: false
1215512155
supportsDBT: false
1215612156
supported_destination_sync_modes: []
12157-
- dockerImage: "airbyte/source-postgres:2.0.25"
12157+
- dockerImage: "airbyte/source-postgres:2.0.26"
1215812158
spec:
1215912159
documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
1216012160
connectionSpecification:

‎airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java

-39
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@
2525
import com.fasterxml.jackson.databind.JsonNode;
2626
import com.google.common.collect.Maps;
2727
import java.sql.JDBCType;
28-
import java.text.CharacterIterator;
29-
import java.text.StringCharacterIterator;
3028
import java.util.HashMap;
3129
import java.util.List;
3230
import java.util.Map;
@@ -137,41 +135,4 @@ public static boolean useSsl(final JsonNode config) {
137135
return config.get(SSL_KEY).asBoolean();
138136
}
139137

140-
/**
141-
* Helper method for logging bytes in a human-readable format. This method logs in SI units
142-
* (B/kB/MB/GB)
143-
*/
144-
public static String humanReadableByteCountSI(final long bytes) {
145-
if (-1000 < bytes && bytes < 1000) {
146-
return bytes + " B";
147-
}
148-
final CharacterIterator ci = new StringCharacterIterator("kMGTPE");
149-
long formattedBytes = bytes;
150-
while (bytes <= -999_950 || bytes >= 999_950) {
151-
formattedBytes /= 1000;
152-
ci.next();
153-
}
154-
return String.format("%.1f %cB", formattedBytes / 1000.0, ci.current());
155-
}
156-
157-
/**
158-
* Helper method for logging bytes in a human-readable format. This method logs in Binary units
159-
* (B/KiB/MiB/GiB)
160-
*/
161-
public static String humanReadableByteCountBin(final long bytes) {
162-
final long absB = bytes == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(bytes);
163-
final int bytePrefixCount = 1024;
164-
if (absB < bytePrefixCount) {
165-
return bytes + " B";
166-
}
167-
long value = absB;
168-
final CharacterIterator ci = new StringCharacterIterator("KMGTPE");
169-
for (int i = 40; i >= 0 && absB > 0xfffccccccccccccL >> i; i -= 10) {
170-
value >>= 10;
171-
ci.next();
172-
}
173-
value *= Long.signum(bytes);
174-
return String.format("%.1f %ciB", value / 1024.0, ci.current());
175-
}
176-
177138
}

‎airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java

+14-17
Original file line numberDiff line numberDiff line change
@@ -442,24 +442,21 @@ protected void logPreSyncDebugData(final JdbcDatabase database, final Configured
442442
database.getMetaData().getDatabaseProductVersion());
443443

444444
for (final ConfiguredAirbyteStream stream : catalog.getStreams()) {
445-
if (stream.getSyncMode().equals(SyncMode.INCREMENTAL)) {
446-
final String streamName = stream.getStream().getName();
447-
final String schemaName = stream.getStream().getNamespace();
448-
final String cursorFieldName = stream.getCursorField() != null && stream.getCursorField().size() != 0 ? stream.getCursorField().get(0) : "";
449-
final ResultSet indexInfo = database.getMetaData().getIndexInfo(null,
450-
schemaName,
451-
streamName,
452-
false,
453-
false);
454-
LOGGER.info("Discovering indexes for schema \"{}\", table \"{}\", with cursor field \"{}\"", schemaName, streamName, cursorFieldName);
455-
while (indexInfo.next()) {
456-
LOGGER.info("Index name: {}, Column: {}, Unique: {}",
457-
indexInfo.getString(JDBC_INDEX_NAME),
458-
indexInfo.getString(JDBC_COLUMN_COLUMN_NAME),
459-
!indexInfo.getBoolean(JDBC_INDEX_NON_UNIQUE));
460-
}
461-
indexInfo.close();
445+
final String streamName = stream.getStream().getName();
446+
final String schemaName = stream.getStream().getNamespace();
447+
final ResultSet indexInfo = database.getMetaData().getIndexInfo(null,
448+
schemaName,
449+
streamName,
450+
false,
451+
false);
452+
LOGGER.info("Discovering indexes for schema \"{}\", table \"{}\"", schemaName, streamName);
453+
while (indexInfo.next()) {
454+
LOGGER.info("Index name: {}, Column: {}, Unique: {}",
455+
indexInfo.getString(JDBC_INDEX_NAME),
456+
indexInfo.getString(JDBC_COLUMN_COLUMN_NAME),
457+
!indexInfo.getBoolean(JDBC_INDEX_NON_UNIQUE));
462458
}
459+
indexInfo.close();
463460
}
464461
}
465462

‎airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=2.0.25
19+
LABEL io.airbyte.version=2.0.26
2020
LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt

‎airbyte-integrations/connectors/source-postgres/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=2.0.25
19+
LABEL io.airbyte.version=2.0.26
2020
LABEL io.airbyte.name=airbyte/source-postgres

‎airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@
7474
import java.sql.Connection;
7575
import java.sql.PreparedStatement;
7676
import java.sql.SQLException;
77-
import java.text.NumberFormat;
7877
import java.time.Duration;
7978
import java.time.Instant;
8079
import java.util.ArrayList;
@@ -573,8 +572,8 @@ protected void estimateFullRefreshSyncSize(final JdbcDatabase database,
573572
// read a row and Stringify it to better understand the accurate volume of data sent over the wire.
574573
// However, this approach doesn't account for different row sizes.
575574
AirbyteTraceMessageUtility.emitEstimateTrace(PLATFORM_DATA_INCREASE_FACTOR * syncByteCount, Type.STREAM, syncRowCount, tableName, schemaName);
576-
LOGGER.info(String.format("Estimate for table in full refresh mode: %s : {rows_to_sync: %s, data_to_sync: %s}",
577-
fullTableName, NumberFormat.getInstance().format(syncRowCount), JdbcUtils.humanReadableByteCountSI(syncByteCount)));
575+
LOGGER.info(String.format("Estimate for table: %s : {sync_row_count: %s, sync_bytes: %s, total_table_row_count: %s, total_table_bytes: %s}",
576+
fullTableName, syncRowCount, syncByteCount, syncRowCount, syncByteCount));
578577
}
579578
} catch (final SQLException e) {
580579
LOGGER.warn("Error occurred while attempting to estimate sync size", e);
@@ -615,9 +614,8 @@ protected void estimateIncrementalSyncSize(final JdbcDatabase database,
615614
// read a row and Stringify it to better understand the accurate volume of data sent over the wire.
616615
// However, this approach doesn't account for different row sizes
617616
AirbyteTraceMessageUtility.emitEstimateTrace(PLATFORM_DATA_INCREASE_FACTOR * syncByteCount, Type.STREAM, syncRowCount, tableName, schemaName);
618-
LOGGER.info(String.format("Estimate for table in incremental mode: %s : {rows_to_sync: %s, data_to_sync: %s, table_row_count: %s, table_size: %s}",
619-
fullTableName, NumberFormat.getInstance().format(syncRowCount), JdbcUtils.humanReadableByteCountSI(syncByteCount), NumberFormat.getInstance().format(tableRowCount),
620-
JdbcUtils.humanReadableByteCountSI(tableByteCount)));
617+
LOGGER.info(String.format("Estimate for table: %s : {sync_row_count: %s, sync_bytes: %s, total_table_row_count: %s, total_table_bytes: %s}",
618+
fullTableName, syncRowCount, syncByteCount, tableRowCount, tableRowCount));
621619
} catch (final SQLException e) {
622620
LOGGER.warn("Error occurred while attempting to estimate sync size", e);
623621
}

‎connectors.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@
183183
| **PokeAPI** | <img alt="PokeAPI icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/pokeapi.svg" height="30" height="30"/> | Source | airbyte/source-pokeapi:0.1.5 | alpha | [docs](https://docs.airbyte.com/integrations/sources/pokeapi) | [connectors/source/pokeapi](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/pokeapi) | [source-pokeapi](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pokeapi) | <small>`6371b14b-bc68-4236-bfbd-468e8df8e968`</small> |
184184
| **Polygon Stock API** | <img alt="Polygon Stock API icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/polygon.svg" height="30" height="30"/> | Source | airbyte/source-polygon-stock-api:0.1.1 | alpha | [docs](https://docs.airbyte.com/integrations/sources/polygon-stock-api) | [connectors/source/polygon-stock-api](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/polygon-stock-api) | [source-polygon-stock-api](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-polygon-stock-api) | <small>`5807d72f-0abc-49f9-8fa5-ae820007032b`</small> |
185185
| **PostHog** | <img alt="PostHog icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/posthog.svg" height="30" height="30"/> | Source | airbyte/source-posthog:0.1.9 | beta | [docs](https://docs.airbyte.com/integrations/sources/posthog) | [connectors/source/posthog](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/posthog) | [source-posthog](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-posthog) | <small>`af6d50ee-dddf-4126-a8ee-7faee990774f`</small> |
186-
| **Postgres** | <img alt="Postgres icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/postgresql.svg" height="30" height="30"/> | Source | airbyte/source-postgres:2.0.25 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/postgres) | [connectors/source/postgres](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/postgres) | [source-postgres](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-postgres) | <small>`decd338e-5647-4c0b-adf4-da0e75f5a750`</small> |
186+
| **Postgres** | <img alt="Postgres icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/postgresql.svg" height="30" height="30"/> | Source | airbyte/source-postgres:2.0.26 | generally_available | [docs](https://docs.airbyte.com/integrations/sources/postgres) | [connectors/source/postgres](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/postgres) | [source-postgres](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-postgres) | <small>`decd338e-5647-4c0b-adf4-da0e75f5a750`</small> |
187187
| **Postmark App** | <img alt="Postmark App icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/postmark.svg" height="30" height="30"/> | Source | airbyte/source-postmarkapp:0.1.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/postmarkapp) | [connectors/source/postmarkapp](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/postmarkapp) | [source-postmarkapp](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-postmarkapp) | <small>`cde75ca1-1e28-4a0f-85bb-90c546de9f1f`</small> |
188188
| **PrestaShop** | <img alt="PrestaShop icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/prestashop.svg" height="30" height="30"/> | Source | airbyte/source-prestashop:0.3.1 | beta | [docs](https://docs.airbyte.com/integrations/sources/prestashop) | [connectors/source/prestashop](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/prestashop) | [source-prestashop](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-prestashop) | <small>`d60a46d4-709f-4092-a6b7-2457f7d455f5`</small> |
189189
| **Primetric** | <img alt="Primetric icon" src="https://raw.githubusercontent.com/airbytehq/airbyte /master/airbyte-config-oss/init-oss/src/main/resources/icons/primetric.svg" height="30" height="30"/> | Source | airbyte/source-primetric:0.1.0 | alpha | [docs](https://docs.airbyte.com/integrations/sources/primetric) | [connectors/source/primetric](https://github.com/airbytehq/airbyte/issues?q=is:open+is:issue+label:connectors/source/primetric) | [source-primetric](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-primetric) | <small>`f636c3c6-4077-45ac-b109-19fc62a283c1`</small> |

‎docs/integrations/sources/postgres.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -399,7 +399,8 @@ Some larger tables may encounter an error related to the temporary file size lim
399399

400400
| Version | Date | Pull Request | Subject |
401401
|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
402-
| 2.0.25 | 2023-04-24 | [25459](https://github.com/airbytehq/airbyte/pull/25459) | Better logging formatting |
402+
| 2.0.26 | 2023-04-26 | [25560](https://github.com/airbytehq/airbyte/pull/25560) | Revert some logging changes |
403+
| 2.0.25 | 2023-04-24 | [25459](https://github.com/airbytehq/airbyte/pull/25459) | Better logging formatting
403404
| 2.0.24 | 2023-04-19 | [25345](https://github.com/airbytehq/airbyte/pull/25345) | Logging : Log database indexes per stream |
404405
| 2.0.23 | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582) | CDC : Enable frequent state emission during incremental syncs + refactor for performance improvement |
405406
| 2.0.22 | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220) | Logging changes : Log additional metadata & clean up noisy logs |

0 commit comments

Comments
 (0)