From 730ce517839e92da754f3be1560fe9e03d65b260 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 19 Jan 2022 11:48:13 +0200 Subject: [PATCH 01/27] airbyte-9328: Added Sentry integration to BigQuery and BigQuery denormalized connector. --- .../Dockerfile | 1 + .../build.gradle | 1 + .../BigQueryDenormalizedDestination.java | 30 +++++++++++++++++-- .../src/main/resources/log4j2.xml | 11 +++++++ .../destination-bigquery/Dockerfile | 1 + .../bigquery/BigQueryDestination.java | 22 ++++++-------- .../src/main/resources/log4j2.xml | 11 +++++++ 7 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/log4j2.xml create mode 100644 airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index be38d6f1cd9df..7316591e959ad 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,6 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized +ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle index 5b0c1bd57006c..92e0343adddec 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle @@ -12,6 +12,7 @@ application { dependencies { implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' implementation 'org.apache.commons:commons-lang3:3.11' + implementation 'io.sentry:sentry-log4j2:5.5.3' implementation project(':airbyte-config:models') implementation project(':airbyte-integrations:bases:base-java') diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java index 4d4f79cc59c80..5f122947bfe61 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java @@ -11,6 +11,9 @@ import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryDenormalizedRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.GcsBigQueryDenormalizedRecordFormatter; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; +import io.sentry.ITransaction; +import io.sentry.Sentry; +import io.sentry.SpanStatus; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,6 +22,18 @@ public class BigQueryDenormalizedDestination extends BigQueryDestination { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedDestination.class); + public static void initSentry() { + Sentry.init(options -> { + // allow setting properties from env variables see https://docs.sentry.io/platforms/java/configuration/ + options.setEnableExternalConfiguration(true); + // To set a uniform sample rate + options.setTracesSampleRate(1.0); + }); + Sentry.configureScope(scope -> { + scope.setTag("connector", "destination-bigquery-denormalized"); + }); + } + @Override protected String getTargetTableName(final String streamName) { // This BigQuery destination does not write to a staging "raw" table but directly to a normalized @@ -46,10 +61,19 @@ protected boolean isDefaultAirbyteTmpTableSchema() { } public static void main(final String[] args) throws Exception { + initSentry(); final Destination destination = new BigQueryDenormalizedDestination(); - LOGGER.info("starting destination: {}", BigQueryDenormalizedDestination.class); - new IntegrationRunner(destination).run(args); - LOGGER.info("completed destination: {}", BigQueryDenormalizedDestination.class); + ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); + try { + LOGGER.info("starting destination: {}", BigQueryDestination.class); + new IntegrationRunner(destination).run(args); + } catch (Exception e) { + transaction.setThrowable(e); + transaction.setStatus(SpanStatus.INTERNAL_ERROR); + } finally { + transaction.finish(); + LOGGER.info("completed destination: {}", BigQueryDestination.class); + } } } diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/log4j2.xml b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/log4j2.xml new file mode 100644 index 0000000000000..8ddbd05927098 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/log4j2.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index e4bb7588c7ca2..73ed893624fe2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,6 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery +ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 4f4c3502afd65..b60c977c74994 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -124,32 +124,29 @@ protected BigQuery getBigQuery(final JsonNode config) { * 1. Create a temporary table for each stream *

*

- * 2. Write records to each stream directly (the bigquery client handles managing when to push the - * records over the network) + * 2. Write records to each stream directly (the bigquery client handles managing when to push the records over the network) *

*

- * 4. Once all records have been written close the writers, so that any remaining records are - * flushed. + * 4. Once all records have been written close the writers, so that any remaining records are flushed. *

*

* 5. Copy the temp tables to the final table name (overwriting if necessary). *

* - * @param config - integration-specific configuration object as json. e.g. { "username": "airbyte", - * "password": "super secure" } + * @param config - integration-specific configuration object as json. e.g. { "username": "airbyte", "password": "super secure" } * @param catalog - schema of the incoming messages. * @return consumer that writes singer messages to the database. */ @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) throws IOException { return getRecordConsumer(getUploaderMap(config, catalog), outputRecordCollector); } protected Map> getUploaderMap(final JsonNode config, - final ConfiguredAirbyteCatalog catalog) + final ConfiguredAirbyteCatalog catalog) throws IOException { final BigQuery bigquery = getBigQuery(config); @@ -176,9 +173,8 @@ protected Map> getUp } /** - * BigQuery might have different structure of the Temporary table. If this method returns TRUE, - * temporary table will have only three common Airbyte attributes. In case of FALSE, temporary table - * structure will be in line with Airbyte message JsonSchema. + * BigQuery might have different structure of the Temporary table. If this method returns TRUE, temporary table will have only three common Airbyte + * attributes. In case of FALSE, temporary table structure will be in line with Airbyte message JsonSchema. * * @return use default AirbyteSchema or build using JsonSchema */ @@ -197,7 +193,7 @@ protected String getTargetTableName(final String streamName) { } protected AirbyteMessageConsumer getRecordConsumer(final Map> writeConfigs, - final Consumer outputRecordCollector) { + final Consumer outputRecordCollector) { return new BigQueryRecordConsumer(writeConfigs, outputRecordCollector); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml new file mode 100644 index 0000000000000..8ddbd05927098 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file From b6fcde369d04888408f1f3769c36e6580e652119 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 19 Jan 2022 11:49:45 +0200 Subject: [PATCH 02/27] airbyte-5050: Added strategy for INSERT ROW. --- .../destination-bigquery/build.gradle | 1 + .../bigquery/BigQueryDestination.java | 29 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index ed1bc14b1a761..360218ca24892 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -12,6 +12,7 @@ application { dependencies { implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' implementation 'org.apache.commons:commons-lang3:3.11' + implementation 'io.sentry:sentry-log4j2:5.5.3' // csv implementation 'org.apache.commons:commons-csv:1.4' diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index b60c977c74994..53b7772610ea8 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -34,6 +34,9 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.sentry.ITransaction; +import io.sentry.Sentry; +import io.sentry.SpanStatus; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.HashMap; @@ -53,6 +56,18 @@ public BigQueryDestination() { namingResolver = new BigQuerySQLNameTransformer(); } + public static void initSentry() { + Sentry.init(options -> { + // allow setting properties from env variables see https://docs.sentry.io/platforms/java/configuration/ + options.setEnableExternalConfiguration(true); + // To set a uniform sample rate + options.setTracesSampleRate(1.0); + }); + Sentry.configureScope(scope -> { + scope.setTag("connector", "destination-bigquery"); + }); + } + @Override public AirbyteConnectionStatus check(final JsonNode config) { try { @@ -198,10 +213,18 @@ protected AirbyteMessageConsumer getRecordConsumer(final Map Date: Mon, 24 Jan 2022 12:06:08 +0200 Subject: [PATCH 03/27] airbyte-9328: Added Sentry integration to Snowflake. --- .../destination-snowflake/Dockerfile | 1 + .../destination-snowflake/build.gradle | 1 + .../snowflake/SnowflakeDatabase.java | 3 +- .../snowflake/SnowflakeDestination.java | 35 +++++++++++++++++-- .../src/main/resources/log4j2.xml | 11 ++++++ 5 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index f854615fb1c72..1f7ab1a0b1dfb 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -11,6 +11,7 @@ WORKDIR /airbyte ENV APPLICATION destination-snowflake +ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" # Needed for JDK17 (in turn, needed on M1 macs) - see https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 ENV DESTINATION_SNOWFLAKE_OPTS "--add-opens java.base/java.nio=ALL-UNNAMED" diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index b9e057e3fa35d..554549237aeda 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -30,6 +30,7 @@ dependencies { implementation 'org.apache.commons:commons-csv:1.4' implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' implementation "io.aesy:datasize:1.0.0" + implementation 'io.sentry:sentry-log4j2:5.5.3' implementation project(':airbyte-config:models') implementation project(':airbyte-db:lib') diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index 76ebade012a98..b5ddf61cc6a12 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -23,7 +23,8 @@ public class SnowflakeDatabase { private static final Duration QUERY_TIMEOUT = Duration.ofHours(3); private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); - public static Connection getConnection(final JsonNode config) throws SQLException { + public static Connection getConnection(final JsonNode config) + throws SQLException { final String connectUrl = String.format("jdbc:snowflake://%s", config.get("host").asText()); final Properties properties = new Properties(); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index 5bf8c04d72e1a..b32419e30fe8a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -9,6 +9,9 @@ import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.destination.jdbc.copy.SwitchingDestination; +import io.sentry.ITransaction; +import io.sentry.Sentry; +import io.sentry.SpanStatus; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +20,18 @@ public class SnowflakeDestination extends SwitchingDestination { + // allow setting properties from env variables see https://docs.sentry.io/platforms/java/configuration/ + options.setEnableExternalConfiguration(true); + // To set a uniform sample rate + options.setTracesSampleRate(1.0); + }); + Sentry.configureScope(scope -> { + scope.setTag("connector", "destination-snowflake"); + }); + } + enum DestinationType { INSERT, COPY_S3, @@ -65,10 +80,24 @@ public static Map getTypeToDestination() { } public static void main(final String[] args) throws Exception { + initSentry(); + try { + throw new Exception("This is hello from Sentry"); + } catch (Exception e) { + LOGGER.error(e.getMessage()); + } final Destination destination = new SnowflakeDestination(); - LOGGER.info("starting destination: {}", SnowflakeDestination.class); - new IntegrationRunner(destination).run(args); - LOGGER.info("completed destination: {}", SnowflakeDestination.class); + ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); + try { + LOGGER.info("starting destination: {}", SnowflakeDestination.class); + new IntegrationRunner(destination).run(args); + } catch (Exception e) { + transaction.setThrowable(e); + transaction.setStatus(SpanStatus.INTERNAL_ERROR); + } finally { + transaction.finish(); + LOGGER.info("completed destination: {}", SnowflakeDestination.class); + } } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml new file mode 100644 index 0000000000000..8ddbd05927098 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file From a0fe903fcc6e09c9c5e1d53f08673bbc5a99f70a Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Tue, 25 Jan 2022 14:40:36 +0200 Subject: [PATCH 04/27] airbyte-9328: Fix Sentry config. --- airbyte-integrations/bases/base-java/Dockerfile | 3 +++ .../connectors/destination-bigquery-denormalized/Dockerfile | 1 - .../connectors/destination-bigquery/Dockerfile | 1 - .../connectors/destination-snowflake/Dockerfile | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/base-java/Dockerfile b/airbyte-integrations/bases/base-java/Dockerfile index b504b46476085..77587aee610a5 100644 --- a/airbyte-integrations/bases/base-java/Dockerfile +++ b/airbyte-integrations/bases/base-java/Dockerfile @@ -13,6 +13,9 @@ ENV AIRBYTE_DISCOVER_CMD "/airbyte/javabase.sh --discover" ENV AIRBYTE_READ_CMD "/airbyte/javabase.sh --read" ENV AIRBYTE_WRITE_CMD "/airbyte/javabase.sh --write" +# Sentry DSN Key +ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" + ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" ENTRYPOINT ["/airbyte/base.sh"] diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 7316591e959ad..be38d6f1cd9df 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,7 +13,6 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized -ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 73ed893624fe2..e4bb7588c7ca2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,7 +13,6 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery -ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 1f7ab1a0b1dfb..f854615fb1c72 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -11,7 +11,6 @@ WORKDIR /airbyte ENV APPLICATION destination-snowflake -ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" # Needed for JDK17 (in turn, needed on M1 macs) - see https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 ENV DESTINATION_SNOWFLAKE_OPTS "--add-opens java.base/java.nio=ALL-UNNAMED" From 460d959956c3e6f3cd2dad67ea24d01b2e455ea4 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 26 Jan 2022 14:52:49 +0200 Subject: [PATCH 05/27] airbyte-9328: Fixed PR comments. --- .env | 3 +++ .env.dev | 3 +-- airbyte-integrations/bases/base-java/Dockerfile | 3 --- .../base-java}/src/main/resources/log4j2.xml | 0 .../src/main/resources/log4j2.xml | 11 ----------- .../connectors/destination-snowflake/build.gradle | 1 - .../src/main/resources/log4j2.xml | 11 ----------- docker-compose.yaml | 1 + 8 files changed, 5 insertions(+), 28 deletions(-) rename airbyte-integrations/{connectors/destination-bigquery-denormalized => bases/base-java}/src/main/resources/log4j2.xml (100%) delete mode 100644 airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml delete mode 100644 airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml diff --git a/.env b/.env index a538aedd6eedc..02e2b99cddf7c 100644 --- a/.env +++ b/.env @@ -88,3 +88,6 @@ MAX_SYNC_WORKERS=5 MAX_SPEC_WORKERS=5 MAX_CHECK_WORKERS=5 MAX_DISCOVER_WORKERS=5 + +# Sentry +JOB_DEFAULT_ENV_SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" \ No newline at end of file diff --git a/.env.dev b/.env.dev index e7a4f02b7d5b0..9e101079a4595 100644 --- a/.env.dev +++ b/.env.dev @@ -26,5 +26,4 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 # Sentry -SENTRY_DSN="" - +JOB_DEFAULT_ENV_SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" \ No newline at end of file diff --git a/airbyte-integrations/bases/base-java/Dockerfile b/airbyte-integrations/bases/base-java/Dockerfile index 77587aee610a5..b504b46476085 100644 --- a/airbyte-integrations/bases/base-java/Dockerfile +++ b/airbyte-integrations/bases/base-java/Dockerfile @@ -13,9 +13,6 @@ ENV AIRBYTE_DISCOVER_CMD "/airbyte/javabase.sh --discover" ENV AIRBYTE_READ_CMD "/airbyte/javabase.sh --read" ENV AIRBYTE_WRITE_CMD "/airbyte/javabase.sh --write" -# Sentry DSN Key -ENV SENTRY_DSN "https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" - ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" ENTRYPOINT ["/airbyte/base.sh"] diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/log4j2.xml b/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml similarity index 100% rename from airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/log4j2.xml rename to airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml deleted file mode 100644 index 8ddbd05927098..0000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/log4j2.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 554549237aeda..b9e057e3fa35d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -30,7 +30,6 @@ dependencies { implementation 'org.apache.commons:commons-csv:1.4' implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' implementation "io.aesy:datasize:1.0.0" - implementation 'io.sentry:sentry-log4j2:5.5.3' implementation project(':airbyte-config:models') implementation project(':airbyte-db:lib') diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml deleted file mode 100644 index 8ddbd05927098..0000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/log4j2.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 6b4e623882394..ccf7616f77d59 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -119,6 +119,7 @@ services: - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT} - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT} - WORKSPACE_ROOT=${WORKSPACE_ROOT} + - JOB_DEFAULT_ENV_SENTRY_DSN=${JOB_DEFAULT_ENV_SENTRY_DSN} volumes: - /var/run/docker.sock:/var/run/docker.sock - workspace:${WORKSPACE_ROOT} From 760fc4200f9b16d998af6964d1bd6bbb6ca280ac Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 26 Jan 2022 14:56:30 +0200 Subject: [PATCH 06/27] airbyte-9328: Fixed PR comments. --- .../connectors/destination-bigquery-denormalized/build.gradle | 1 - .../connectors/destination-bigquery/build.gradle | 1 - 2 files changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle index 92e0343adddec..5b0c1bd57006c 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle @@ -12,7 +12,6 @@ application { dependencies { implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'io.sentry:sentry-log4j2:5.5.3' implementation project(':airbyte-config:models') implementation project(':airbyte-integrations:bases:base-java') diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 360218ca24892..ed1bc14b1a761 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -12,7 +12,6 @@ application { dependencies { implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'io.sentry:sentry-log4j2:5.5.3' // csv implementation 'org.apache.commons:commons-csv:1.4' From cdc47d84c1d85ef6b21d37052bafb0c2cec15a55 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 26 Jan 2022 14:58:10 +0200 Subject: [PATCH 07/27] airbyte-9328: Fix PR comments. --- airbyte-config/models/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-config/models/build.gradle b/airbyte-config/models/build.gradle index d62c88c7e1633..dcadb7ad7a555 100644 --- a/airbyte-config/models/build.gradle +++ b/airbyte-config/models/build.gradle @@ -5,6 +5,8 @@ plugins { } dependencies { + // https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-api + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' implementation project(':airbyte-json-validation') implementation project(':airbyte-protocol:models') implementation project(':airbyte-commons') From ba11f20f959f577c34b7d7d23ba6484a4da03ac2 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 26 Jan 2022 15:04:00 +0200 Subject: [PATCH 08/27] airbyte-9328: Fixed PR comments. --- airbyte-config/models/build.gradle | 2 -- airbyte-integrations/bases/base-java/build.gradle | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-config/models/build.gradle b/airbyte-config/models/build.gradle index dcadb7ad7a555..d62c88c7e1633 100644 --- a/airbyte-config/models/build.gradle +++ b/airbyte-config/models/build.gradle @@ -5,8 +5,6 @@ plugins { } dependencies { - // https://mvnrepository.com/artifact/org.junit.jupiter/junit-jupiter-api - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' implementation project(':airbyte-json-validation') implementation project(':airbyte-protocol:models') implementation project(':airbyte-commons') diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 136e0f8c19d49..5795a884f7ef3 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -7,6 +7,9 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-commons-cli') implementation project(':airbyte-json-validation') + implementation 'io.sentry:sentry-log4j2:5.6.0' +// https://mvnrepository.com/artifact/org.junit.vintage/junit-vintage-engine + testImplementation 'org.junit.vintage:junit-vintage-engine:5.8.2' implementation 'commons-cli:commons-cli:1.4' implementation 'org.apache.sshd:sshd-mina:2.7.0' From ed047b431efd8480db4f36d9dc3ce8fa44719bfc Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 26 Jan 2022 15:04:38 +0200 Subject: [PATCH 09/27] airbyte-9328: Fixed PR comments. --- airbyte-integrations/bases/base-java/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 5795a884f7ef3..293e93dbf2135 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -8,8 +8,6 @@ dependencies { implementation project(':airbyte-commons-cli') implementation project(':airbyte-json-validation') implementation 'io.sentry:sentry-log4j2:5.6.0' -// https://mvnrepository.com/artifact/org.junit.vintage/junit-vintage-engine - testImplementation 'org.junit.vintage:junit-vintage-engine:5.8.2' implementation 'commons-cli:commons-cli:1.4' implementation 'org.apache.sshd:sshd-mina:2.7.0' From 0f5b52194e3d1c11dc12757de9380188623596c6 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Wed, 26 Jan 2022 17:08:01 +0200 Subject: [PATCH 10/27] airbyte-9328: Fixed PR comments. --- .env.dev | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.dev b/.env.dev index 9e101079a4595..04622798bd9f4 100644 --- a/.env.dev +++ b/.env.dev @@ -26,4 +26,5 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 # Sentry -JOB_DEFAULT_ENV_SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" \ No newline at end of file +SENTRY_DSN="" +JOB_DEFAULT_ENV_SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" From ad1824d37bd9e475776e4aeae026ec7bf81a419f Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Thu, 27 Jan 2022 17:29:11 +0200 Subject: [PATCH 11/27] airbyte-9328: Small changes. --- .../destination/bigquery/BigQueryDestination.java | 1 + .../destination/snowflake/SnowflakeDestination.java | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 53b7772610ea8..bf5cbc0542758 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -213,6 +213,7 @@ protected AirbyteMessageConsumer getRecordConsumer(final Map getTypeToDestination() { public static void main(final String[] args) throws Exception { initSentry(); - try { - throw new Exception("This is hello from Sentry"); - } catch (Exception e) { - LOGGER.error(e.getMessage()); - } final Destination destination = new SnowflakeDestination(); ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); try { From 000328f8fb5d9214a2bb9d90d9040684b3ba5693 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Thu, 27 Jan 2022 18:59:01 +0200 Subject: [PATCH 12/27] airbyte-9328: Small changes. --- airbyte-integrations/bases/base-java/build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 293e93dbf2135..502e42ca96311 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -7,7 +7,8 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-commons-cli') implementation project(':airbyte-json-validation') - implementation 'io.sentry:sentry-log4j2:5.6.0' + api 'io.sentry:sentry-log4j2:5.6.0' + api 'io.sentry:sentry:5.6.0' implementation 'commons-cli:commons-cli:1.4' implementation 'org.apache.sshd:sshd-mina:2.7.0' @@ -21,4 +22,4 @@ dependencies { implementation "org.testcontainers:jdbc:1.15.3" implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) -} +} \ No newline at end of file From a89eb7d81fd15fdc3bcf1e8ec7641ef1b822a2c7 Mon Sep 17 00:00:00 2001 From: Oleksandr Tsukanov Date: Fri, 28 Jan 2022 14:14:12 +0200 Subject: [PATCH 13/27] airbyte-9328: Move SENTRY DSN keys to Dockerfiles. --- .env | 5 +---- .env.dev | 3 +-- .../connectors/destination-bigquery-denormalized/Dockerfile | 1 + .../connectors/destination-bigquery/Dockerfile | 1 + .../connectors/destination-snowflake/Dockerfile | 2 ++ 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 02e2b99cddf7c..942532cdc03ce 100644 --- a/.env +++ b/.env @@ -87,7 +87,4 @@ SUBMITTER_NUM_THREADS=10 MAX_SYNC_WORKERS=5 MAX_SPEC_WORKERS=5 MAX_CHECK_WORKERS=5 -MAX_DISCOVER_WORKERS=5 - -# Sentry -JOB_DEFAULT_ENV_SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" \ No newline at end of file +MAX_DISCOVER_WORKERS=5 \ No newline at end of file diff --git a/.env.dev b/.env.dev index 04622798bd9f4..93f7274526c7f 100644 --- a/.env.dev +++ b/.env.dev @@ -26,5 +26,4 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 # Sentry -SENTRY_DSN="" -JOB_DEFAULT_ENV_SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" +SENTRY_DSN="" \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index be38d6f1cd9df..62073d4304b65 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,6 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized +ENV SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index e4bb7588c7ca2..4ad8e5f36fe9d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,6 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery +ENV SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index f854615fb1c72..9e3d68430bb6b 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,5 +18,7 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 +ENV SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" + LABEL io.airbyte.version=0.4.2 LABEL io.airbyte.name=airbyte/destination-snowflake From 2881b0e2f8bb16e811b8d3784677cff6de301a30 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Fri, 28 Jan 2022 17:50:06 -0800 Subject: [PATCH 14/27] Use new dsn --- .env.dev | 2 +- airbyte-integrations/bases/base-java/build.gradle | 2 +- .../bases/base-java/src/main/resources/log4j2.xml | 2 +- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- .../connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- .../integrations/destination/snowflake/SnowflakeDatabase.java | 3 +-- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.env.dev b/.env.dev index 93f7274526c7f..2a6dc7eb8129b 100644 --- a/.env.dev +++ b/.env.dev @@ -26,4 +26,4 @@ SYNC_JOB_MAX_ATTEMPTS=3 SYNC_JOB_MAX_TIMEOUT_DAYS=3 # Sentry -SENTRY_DSN="" \ No newline at end of file +SENTRY_DSN="" diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 502e42ca96311..b18123cbf2446 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -22,4 +22,4 @@ dependencies { implementation "org.testcontainers:jdbc:1.15.3" implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) -} \ No newline at end of file +} diff --git a/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml b/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml index 8ddbd05927098..26e90bb09e454 100644 --- a/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml +++ b/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml @@ -8,4 +8,4 @@ - \ No newline at end of file + diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 3cd32422c2873..beb6b4aebbe6e 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,7 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized -ENV SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" +ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index f149f03db1850..df3101a7bbd05 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,7 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery -ENV SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" +ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 5bd996dae2064..b64081b37f198 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,7 +18,7 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -ENV SENTRY_DSN="https://2a3a1622a2104b8f8d36db4915c956f8@o1114644.ingest.sentry.io/6145965" +ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" LABEL io.airbyte.version=0.4.5 diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index b5ddf61cc6a12..76ebade012a98 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -23,8 +23,7 @@ public class SnowflakeDatabase { private static final Duration QUERY_TIMEOUT = Duration.ofHours(3); private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); - public static Connection getConnection(final JsonNode config) - throws SQLException { + public static Connection getConnection(final JsonNode config) throws SQLException { final String connectUrl = String.format("jdbc:snowflake://%s", config.get("host").asText()); final Properties properties = new Properties(); From 4c0fa25cce7c5d22080a9cfa7e952e6008061e7c Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Fri, 28 Jan 2022 18:09:23 -0800 Subject: [PATCH 15/27] Revert format --- .../BigQueryDenormalizedDestination.java | 3 ++- .../bigquery/BigQueryDestination.java | 25 +++++++++++-------- .../snowflake/SnowflakeDestination.java | 3 ++- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java index 5f122947bfe61..ffa111debcf69 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java @@ -24,7 +24,8 @@ public class BigQueryDenormalizedDestination extends BigQueryDestination { public static void initSentry() { Sentry.init(options -> { - // allow setting properties from env variables see https://docs.sentry.io/platforms/java/configuration/ + // allow setting properties from env variables see + // https://docs.sentry.io/platforms/java/configuration/ options.setEnableExternalConfiguration(true); // To set a uniform sample rate options.setTracesSampleRate(1.0); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index bd5868df4e92b..2a0af6015f233 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -58,7 +58,8 @@ public BigQueryDestination() { public static void initSentry() { Sentry.init(options -> { - // allow setting properties from env variables see https://docs.sentry.io/platforms/java/configuration/ + // allow setting properties from env variables see + // https://docs.sentry.io/platforms/java/configuration/ options.setEnableExternalConfiguration(true); // To set a uniform sample rate options.setTracesSampleRate(1.0); @@ -139,29 +140,32 @@ protected BigQuery getBigQuery(final JsonNode config) { * 1. Create a temporary table for each stream *

*

- * 2. Write records to each stream directly (the bigquery client handles managing when to push the records over the network) + * 2. Write records to each stream directly (the bigquery client handles managing when to push the + * records over the network) *

*

- * 4. Once all records have been written close the writers, so that any remaining records are flushed. + * 4. Once all records have been written close the writers, so that any remaining records are + * flushed. *

*

* 5. Copy the temp tables to the final table name (overwriting if necessary). *

* - * @param config - integration-specific configuration object as json. e.g. { "username": "airbyte", "password": "super secure" } + * @param config - integration-specific configuration object as json. e.g. { "username": "airbyte", + * "password": "super secure" } * @param catalog - schema of the incoming messages. * @return consumer that writes singer messages to the database. */ @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) throws IOException { return getRecordConsumer(getUploaderMap(config, catalog), outputRecordCollector); } protected Map> getUploaderMap(final JsonNode config, - final ConfiguredAirbyteCatalog catalog) + final ConfiguredAirbyteCatalog catalog) throws IOException { final BigQuery bigquery = getBigQuery(config); @@ -188,8 +192,9 @@ protected Map> getUp } /** - * BigQuery might have different structure of the Temporary table. If this method returns TRUE, temporary table will have only three common Airbyte - * attributes. In case of FALSE, temporary table structure will be in line with Airbyte message JsonSchema. + * BigQuery might have different structure of the Temporary table. If this method returns TRUE, + * temporary table will have only three common Airbyte attributes. In case of FALSE, temporary table + * structure will be in line with Airbyte message JsonSchema. * * @return use default AirbyteSchema or build using JsonSchema */ @@ -208,7 +213,7 @@ protected String getTargetTableName(final String streamName) { } protected AirbyteMessageConsumer getRecordConsumer(final Map> writeConfigs, - final Consumer outputRecordCollector) { + final Consumer outputRecordCollector) { return new BigQueryRecordConsumer(writeConfigs, outputRecordCollector); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index 91207d5a11100..1270960f349a0 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -22,7 +22,8 @@ public class SnowflakeDestination extends SwitchingDestination { - // allow setting properties from env variables see https://docs.sentry.io/platforms/java/configuration/ + // allow setting properties from env variables see + // https://docs.sentry.io/platforms/java/configuration/ options.setEnableExternalConfiguration(true); // To set a uniform sample rate options.setTracesSampleRate(1.0); From 9207afbcf5913b9f377d404c88d43220dbfaa2e7 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Fri, 28 Jan 2022 18:10:48 -0800 Subject: [PATCH 16/27] Remove sentry dsn from compose temporarily --- docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 90892b93f7c73..0eccd58cc660d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -120,7 +120,6 @@ services: - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT} - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT} - WORKSPACE_ROOT=${WORKSPACE_ROOT} - - JOB_DEFAULT_ENV_SENTRY_DSN=${JOB_DEFAULT_ENV_SENTRY_DSN} volumes: - /var/run/docker.sock:/var/run/docker.sock - workspace:${WORKSPACE_ROOT} From a9b7fe571ea7add08add74f3a2e531e7980c341b Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Fri, 28 Jan 2022 20:05:57 -0800 Subject: [PATCH 17/27] Log sentry event id --- .../snowflake/SnowflakeDestination.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index 1270960f349a0..bc9217498e57a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -28,9 +28,7 @@ public static void initSentry() { // To set a uniform sample rate options.setTracesSampleRate(1.0); }); - Sentry.configureScope(scope -> { - scope.setTag("connector", "destination-snowflake"); - }); + Sentry.configureScope(scope -> scope.setTag("connector", "destination-snowflake")); } enum DestinationType { @@ -54,7 +52,7 @@ public static DestinationType getTypeFromConfig(final JsonNode config) { } } - public static boolean isInternalStaging(JsonNode config) { + public static boolean isInternalStaging(final JsonNode config) { return config.has("loading_method") && config.get("loading_method").isObject() && config.get("loading_method").get("method").asText().equals("Internal Staging"); } @@ -80,19 +78,19 @@ public static Map getTypeToDestination() { DestinationType.INTERNAL_STAGING, internalStagingDestination); } - public static void main(final String[] args) throws Exception { + public static void main(final String[] args) { initSentry(); final Destination destination = new SnowflakeDestination(); - ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); + final ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); try { - LOGGER.info("starting destination: {}", SnowflakeDestination.class); + LOGGER.info("Starting destination: {} (Sentry event ID {})", SnowflakeDestination.class, transaction.getEventId()); new IntegrationRunner(destination).run(args); - } catch (Exception e) { + } catch (final Exception e) { transaction.setThrowable(e); transaction.setStatus(SpanStatus.INTERNAL_ERROR); } finally { transaction.finish(); - LOGGER.info("completed destination: {}", SnowflakeDestination.class); + LOGGER.info("Completed destination: {}", SnowflakeDestination.class); } } From 74f20d0b31520d8bd2cf1d2fff52cf59e2e567c5 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 01:43:00 -0800 Subject: [PATCH 18/27] Move sentry to java base --- .../bases/base-java/Dockerfile | 2 + .../bases/base-java/build.gradle | 1 - .../integrations/base/IntegrationRunner.java | 43 +++++++++++++- .../base/sentry/AirbyteSentry.java | 56 +++++++++++++++++++ .../base-java/src/main/resources/log4j2.xml | 11 ---- .../destination-e2e-test/Dockerfile | 4 +- .../destination-snowflake/Dockerfile | 6 +- .../snowflake/SnowflakeDestination.java | 29 +--------- .../SnowflakeInternalStagingDestination.java | 23 ++++---- .../connectors/source-e2e-test/Dockerfile | 4 +- 10 files changed, 124 insertions(+), 55 deletions(-) create mode 100644 airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java delete mode 100644 airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml diff --git a/airbyte-integrations/bases/base-java/Dockerfile b/airbyte-integrations/bases/base-java/Dockerfile index b504b46476085..ea1844de4e642 100644 --- a/airbyte-integrations/bases/base-java/Dockerfile +++ b/airbyte-integrations/bases/base-java/Dockerfile @@ -13,6 +13,8 @@ ENV AIRBYTE_DISCOVER_CMD "/airbyte/javabase.sh --discover" ENV AIRBYTE_READ_CMD "/airbyte/javabase.sh --read" ENV AIRBYTE_WRITE_CMD "/airbyte/javabase.sh --write" +ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" + ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" ENTRYPOINT ["/airbyte/base.sh"] diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index b18123cbf2446..2ba85b21ccd79 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -7,7 +7,6 @@ dependencies { implementation project(':airbyte-protocol:models') implementation project(':airbyte-commons-cli') implementation project(':airbyte-json-validation') - api 'io.sentry:sentry-log4j2:5.6.0' api 'io.sentry:sentry:5.6.0' implementation 'commons-cli:commons-cli:1.4' diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java index c0ff0297ee2a8..ef5dbbd876059 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java @@ -15,6 +15,9 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.validation.json.JsonSchemaValidator; +import io.sentry.ITransaction; +import io.sentry.Sentry; +import io.sentry.SpanStatus; import java.nio.file.Path; import java.util.Optional; import java.util.Scanner; @@ -73,10 +76,33 @@ public IntegrationRunner(final Source source) { } public void run(final String[] args) throws Exception { - LOGGER.info("Running integration: {}", integration.getClass().getName()); + initSentry(); final IntegrationConfig parsed = cliParser.parse(args); + final ITransaction transaction = Sentry.startTransaction( + integration.getClass().getSimpleName(), + parsed.getCommand().toString(), + true); + try { + runInternal(transaction, parsed); + transaction.finish(SpanStatus.OK); + } catch (final Exception e) { + LOGGER.error("Connector failure", e); + transaction.setThrowable(e); + transaction.finish(SpanStatus.INTERNAL_ERROR); + throw e; + } finally { + /* + * This finally block may not run, probably because the container + * can be terminated by the worker. So the transaction should always + * be finished in the try and catch blocks. + */ + transaction.finish(); + } + } + public void runInternal(final ITransaction transaction, final IntegrationConfig parsed) throws Exception { + LOGGER.info("Running integration: {}", integration.getClass().getName()); LOGGER.info("Command: {}", parsed.getCommand()); LOGGER.info("Integration config: {}", parsed); @@ -169,4 +195,19 @@ private static T parseConfig(final Path path, final Class klass) { return Jsons.object(jsonNode, klass); } + private static void initSentry() { + final String connector = System.getenv("APPLICATION"); + final String version = System.getenv("APPLICATION_VERSION"); + final boolean enableSentry = Boolean.parseBoolean(System.getenv("ENABLE_SENTRY")); + + // https://docs.sentry.io/platforms/java/configuration/ + Sentry.init(options -> { + options.setEnableExternalConfiguration(true); + options.setTracesSampleRate(enableSentry ? 1.0 : 0.0); + options.setRelease(String.format("airbyte-%s:%s", connector, version)); + options.setTag("connector", connector); + options.setTag("connector_version", version); + }); + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java new file mode 100644 index 0000000000000..15f59f0dea3ae --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java @@ -0,0 +1,56 @@ +package io.airbyte.integrations.base.sentry; + +import io.sentry.ISpan; +import io.sentry.Sentry; +import io.sentry.SpanStatus; +import java.util.concurrent.Callable; + +public class AirbyteSentry { + + @FunctionalInterface + public interface ThrowingRunnable { + void call() throws Exception; + } + + public static void runWithSpan(final String operation, final ThrowingRunnable command) throws Exception { + final ISpan span = Sentry.getSpan(); + final ISpan childSpan; + if (span == null) { + childSpan = Sentry.startTransaction("ROOT", operation); + } else { + childSpan = span.startChild(operation); + } + try { + command.call(); + childSpan.finish(SpanStatus.OK); + } catch (final Exception e) { + childSpan.setThrowable(e); + childSpan.finish(SpanStatus.INTERNAL_ERROR); + throw e; + } finally { + childSpan.finish(); + } + } + + public static T runWithSpan(final String operation, final Callable command) throws Exception { + final ISpan span = Sentry.getSpan(); + final ISpan childSpan; + if (span == null) { + childSpan = Sentry.startTransaction("ROOT", operation); + } else { + childSpan = span.startChild(operation); + } + try { + final T result = command.call(); + childSpan.finish(SpanStatus.OK); + return result; + } catch (final Exception e) { + childSpan.setThrowable(e); + childSpan.finish(SpanStatus.INTERNAL_ERROR); + throw e; + } finally { + childSpan.finish(); + } + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml b/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml deleted file mode 100644 index 26e90bb09e454..0000000000000 --- a/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index 0481fb380af1c..61268bf63988d 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-e2e-test +ENV APPLICATION_VERSION 0.2.1 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/destination-e2e-test diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index b64081b37f198..4a3af2d676b4b 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,8 +18,8 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" - -LABEL io.airbyte.version=0.4.5 +ENV APPLICATION_VERSION 0.4.5 +ENV ENABLE_SENTRY true +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index bc9217498e57a..e0a8bbd110322 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -9,9 +9,6 @@ import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.destination.jdbc.copy.SwitchingDestination; -import io.sentry.ITransaction; -import io.sentry.Sentry; -import io.sentry.SpanStatus; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,17 +17,6 @@ public class SnowflakeDestination extends SwitchingDestination { - // allow setting properties from env variables see - // https://docs.sentry.io/platforms/java/configuration/ - options.setEnableExternalConfiguration(true); - // To set a uniform sample rate - options.setTracesSampleRate(1.0); - }); - Sentry.configureScope(scope -> scope.setTag("connector", "destination-snowflake")); - } - enum DestinationType { INSERT, COPY_S3, @@ -78,20 +64,9 @@ public static Map getTypeToDestination() { DestinationType.INTERNAL_STAGING, internalStagingDestination); } - public static void main(final String[] args) { - initSentry(); + public static void main(final String[] args) throws Exception { final Destination destination = new SnowflakeDestination(); - final ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); - try { - LOGGER.info("Starting destination: {} (Sentry event ID {})", SnowflakeDestination.class, transaction.getEventId()); - new IntegrationRunner(destination).run(args); - } catch (final Exception e) { - transaction.setThrowable(e); - transaction.setStatus(SpanStatus.INTERNAL_ERROR); - } finally { - transaction.finish(); - LOGGER.info("Completed destination: {}", SnowflakeDestination.class); - } + new IntegrationRunner(destination).run(args); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index e1db16e984a31..74deface3a90c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -9,6 +9,7 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.sentry.AirbyteSentry; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteMessage; @@ -27,13 +28,15 @@ public SnowflakeInternalStagingDestination() { } @Override - public AirbyteConnectionStatus check(JsonNode config) { - SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); - SnowflakeStagingSqlOperations snowflakeStagingSqlOperations = new SnowflakeStagingSqlOperations(); + public AirbyteConnectionStatus check(final JsonNode config) { + final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer(); + final SnowflakeStagingSqlOperations snowflakeStagingSqlOperations = new SnowflakeStagingSqlOperations(); try (final JdbcDatabase database = getDatabase(config)) { final String outputSchema = super.getNamingResolver().getIdentifier(config.get("schema").asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations); - attemptSQLCreateAndDropStages(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations); + AirbyteSentry.runWithSpan("CreateAndDropTable", () -> + attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations)); + AirbyteSentry.runWithSpan("CreateAndDropStage", () -> + attemptSQLCreateAndDropStages(outputSchema, database, nameTransformer, snowflakeStagingSqlOperations)); return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); } catch (final Exception e) { LOGGER.error("Exception while checking connection: ", e); @@ -43,15 +46,15 @@ public AirbyteConnectionStatus check(JsonNode config) { } } - private static void attemptSQLCreateAndDropStages(String outputSchema, - JdbcDatabase database, - SnowflakeSQLNameTransformer namingResolver, - SnowflakeStagingSqlOperations sqlOperations) + private static void attemptSQLCreateAndDropStages(final String outputSchema, + final JdbcDatabase database, + final SnowflakeSQLNameTransformer namingResolver, + final SnowflakeStagingSqlOperations sqlOperations) throws Exception { // verify we have permissions to create/drop stage final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); - String stageName = namingResolver.getStageName(outputSchema, outputTableName);; + final String stageName = namingResolver.getStageName(outputSchema, outputTableName);; sqlOperations.createStageIfNotExists(database, stageName); sqlOperations.dropStageIfExists(database, stageName); } diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index 6ba25c8c6f25f..3460036b27021 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION source-e2e-test +ENV APPLICATION_VERSION=1.0.0 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/source-e2e-test From 6cb7ce25fd233978f5d328acc4bebed2df69d979 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 01:49:16 -0800 Subject: [PATCH 19/27] Remove sentry code from bigquery --- .../integrations/base/IntegrationRunner.java | 1 - .../BigQueryDenormalizedDestination.java | 31 ++--------------- .../bigquery/BigQueryDestination.java | 33 ++----------------- 3 files changed, 5 insertions(+), 60 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java index ef5dbbd876059..6c47bce4af84a 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java @@ -87,7 +87,6 @@ public void run(final String[] args) throws Exception { runInternal(transaction, parsed); transaction.finish(SpanStatus.OK); } catch (final Exception e) { - LOGGER.error("Connector failure", e); transaction.setThrowable(e); transaction.finish(SpanStatus.INTERNAL_ERROR); throw e; diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java index ffa111debcf69..cdc862ee31abd 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java @@ -11,9 +11,6 @@ import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryDenormalizedRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.GcsBigQueryDenormalizedRecordFormatter; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; -import io.sentry.ITransaction; -import io.sentry.Sentry; -import io.sentry.SpanStatus; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,19 +19,6 @@ public class BigQueryDenormalizedDestination extends BigQueryDestination { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedDestination.class); - public static void initSentry() { - Sentry.init(options -> { - // allow setting properties from env variables see - // https://docs.sentry.io/platforms/java/configuration/ - options.setEnableExternalConfiguration(true); - // To set a uniform sample rate - options.setTracesSampleRate(1.0); - }); - Sentry.configureScope(scope -> { - scope.setTag("connector", "destination-bigquery-denormalized"); - }); - } - @Override protected String getTargetTableName(final String streamName) { // This BigQuery destination does not write to a staging "raw" table but directly to a normalized @@ -43,7 +27,7 @@ protected String getTargetTableName(final String streamName) { } @Override - protected Map getFormatterMap(JsonNode jsonSchema) { + protected Map getFormatterMap(final JsonNode jsonSchema) { return Map.of(UploaderType.STANDARD, new DefaultBigQueryDenormalizedRecordFormatter(jsonSchema, getNamingResolver()), UploaderType.AVRO, new GcsBigQueryDenormalizedRecordFormatter(jsonSchema, getNamingResolver())); } @@ -62,19 +46,8 @@ protected boolean isDefaultAirbyteTmpTableSchema() { } public static void main(final String[] args) throws Exception { - initSentry(); final Destination destination = new BigQueryDenormalizedDestination(); - ITransaction transaction = Sentry.startTransaction("IntegrationRunner()", "run"); - try { - LOGGER.info("starting destination: {}", BigQueryDestination.class); - new IntegrationRunner(destination).run(args); - } catch (Exception e) { - transaction.setThrowable(e); - transaction.setStatus(SpanStatus.INTERNAL_ERROR); - } finally { - transaction.finish(); - LOGGER.info("completed destination: {}", BigQueryDestination.class); - } + new IntegrationRunner(destination).run(args); } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 2a0af6015f233..b90fc557dccc1 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -34,9 +34,6 @@ import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.sentry.ITransaction; -import io.sentry.Sentry; -import io.sentry.SpanStatus; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.HashMap; @@ -56,19 +53,6 @@ public BigQueryDestination() { namingResolver = new BigQuerySQLNameTransformer(); } - public static void initSentry() { - Sentry.init(options -> { - // allow setting properties from env variables see - // https://docs.sentry.io/platforms/java/configuration/ - options.setEnableExternalConfiguration(true); - // To set a uniform sample rate - options.setTracesSampleRate(1.0); - }); - Sentry.configureScope(scope -> { - scope.setTag("connector", "destination-bigquery"); - }); - } - @Override public AirbyteConnectionStatus check(final JsonNode config) { try { @@ -173,7 +157,7 @@ protected Map> getUp for (final ConfiguredAirbyteStream configStream : catalog.getStreams()) { final AirbyteStream stream = configStream.getStream(); final String streamName = stream.getName(); - UploaderConfig uploaderConfig = UploaderConfig + final UploaderConfig uploaderConfig = UploaderConfig .builder() .bigQuery(bigquery) .configStream(configStream) @@ -202,7 +186,7 @@ protected boolean isDefaultAirbyteTmpTableSchema() { return true; } - protected Map getFormatterMap(JsonNode jsonSchema) { + protected Map getFormatterMap(final JsonNode jsonSchema) { return Map.of(UploaderType.STANDARD, new DefaultBigQueryRecordFormatter(jsonSchema, getNamingResolver()), UploaderType.CSV, new GcsCsvBigQueryRecordFormatter(jsonSchema, getNamingResolver()), UploaderType.AVRO, new GcsAvroBigQueryRecordFormatter(jsonSchema, getNamingResolver())); @@ -218,19 +202,8 @@ protected AirbyteMessageConsumer getRecordConsumer(final Map Date: Sat, 29 Jan 2022 01:50:28 -0800 Subject: [PATCH 20/27] Update dockerfiles --- .../connectors/destination-bigquery-denormalized/Dockerfile | 5 +++-- .../connectors/destination-bigquery/Dockerfile | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index beb6b4aebbe6e..a49c39d06c347 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,9 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized -ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" +ENV APPLICATION_VERSION 0.2.5 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index df3101a7bbd05..85095a8b5bc95 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,9 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery -ENV SENTRY_DSN="https://981e729cf92840628b29121e96e958f7@o1009025.ingest.sentry.io/6173659" +ENV APPLICATION_VERSION 0.6.5 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.5 +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/destination-bigquery From 7bf53a283c4ef0e217ab43d2a353757062157589 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 02:19:31 -0800 Subject: [PATCH 21/27] Fix build --- .../integrations/base/IntegrationRunner.java | 14 ++++++++------ .../integrations/base/sentry/AirbyteSentry.java | 6 ++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java index 6c47bce4af84a..cf8e47798d7ca 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java @@ -19,6 +19,7 @@ import io.sentry.Sentry; import io.sentry.SpanStatus; import java.nio.file.Path; +import java.util.Map; import java.util.Optional; import java.util.Scanner; import java.util.Set; @@ -92,9 +93,8 @@ public void run(final String[] args) throws Exception { throw e; } finally { /* - * This finally block may not run, probably because the container - * can be terminated by the worker. So the transaction should always - * be finished in the try and catch blocks. + * This finally block may not run, probably because the container can be terminated by the worker. + * So the transaction should always be finished in the try and catch blocks. */ transaction.finish(); } @@ -195,12 +195,14 @@ private static T parseConfig(final Path path, final Class klass) { } private static void initSentry() { - final String connector = System.getenv("APPLICATION"); - final String version = System.getenv("APPLICATION_VERSION"); - final boolean enableSentry = Boolean.parseBoolean(System.getenv("ENABLE_SENTRY")); + final Map env = System.getenv(); + final String connector = env.getOrDefault("APPLICATION", "unknown"); + final String version = env.getOrDefault("APPLICATION_VERSION", "unknown"); + final boolean enableSentry = Boolean.parseBoolean(env.getOrDefault("ENABLE_SENTRY", "false")); // https://docs.sentry.io/platforms/java/configuration/ Sentry.init(options -> { + options.setDsn(env.getOrDefault("SENTRY_DSN", "")); options.setEnableExternalConfiguration(true); options.setTracesSampleRate(enableSentry ? 1.0 : 0.0); options.setRelease(String.format("airbyte-%s:%s", connector, version)); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java index 15f59f0dea3ae..9c512e6273056 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/sentry/AirbyteSentry.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.base.sentry; import io.sentry.ISpan; @@ -9,7 +13,9 @@ public class AirbyteSentry { @FunctionalInterface public interface ThrowingRunnable { + void call() throws Exception; + } public static void runWithSpan(final String operation, final ThrowingRunnable command) throws Exception { From 304520bb3ca0846f8157f93817fe3726ea01cd16 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 02:49:20 -0800 Subject: [PATCH 22/27] Update release tag format --- .../java/io/airbyte/integrations/base/IntegrationRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java index cf8e47798d7ca..2e0aa336595d2 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java @@ -205,7 +205,7 @@ private static void initSentry() { options.setDsn(env.getOrDefault("SENTRY_DSN", "")); options.setEnableExternalConfiguration(true); options.setTracesSampleRate(enableSentry ? 1.0 : 0.0); - options.setRelease(String.format("airbyte-%s:%s", connector, version)); + options.setRelease(String.format("%s@%s", connector, version)); options.setTag("connector", connector); options.setTag("connector_version", version); }); From 9ba3d2f5dea79de80235c464449fed97b26157d3 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 02:59:10 -0800 Subject: [PATCH 23/27] Bump version --- .../destination-bigquery-denormalized/Dockerfile | 2 +- .../connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-dev-null/Dockerfile | 2 +- .../connectors/destination-e2e-test/Dockerfile | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- .../connectors/source-e2e-test-cloud/Dockerfile | 2 +- .../connectors/source-e2e-test/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 2 ++ docs/integrations/destinations/e2e-test.md | 10 ++-------- docs/integrations/destinations/snowflake.md | 1 + docs/integrations/sources/e2e-test.md | 11 +++-------- 11 files changed, 15 insertions(+), 23 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index a49c39d06c347..524e7789a7bca 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -13,7 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery-denormalized -ENV APPLICATION_VERSION 0.2.5 +ENV APPLICATION_VERSION 0.2.6 ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 85095a8b5bc95..b89d6cc69e9ab 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -13,7 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-bigquery -ENV APPLICATION_VERSION 0.6.5 +ENV APPLICATION_VERSION 0.6.6 ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile index ef4cab197c2e6..28a246b76686e 100644 --- a/airbyte-integrations/connectors/destination-dev-null/Dockerfile +++ b/airbyte-integrations/connectors/destination-dev-null/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-dev-null COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index 61268bf63988d..86b16b0d7c8fc 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -13,7 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-e2e-test -ENV APPLICATION_VERSION 0.2.1 +ENV APPLICATION_VERSION 0.2.2 ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index 4e7a6f1ad320a..a4626f14709c8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -18,7 +18,7 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -ENV APPLICATION_VERSION 0.4.6 +ENV APPLICATION_VERSION 0.4.7 ENV ENABLE_SENTRY true LABEL io.airbyte.version=${APPLICATION_VERSION} diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile index a230cc6117334..57ae06e025587 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-e2e-test-cloud COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-e2e-test-cloud diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index 3460036b27021..1f3509aeeb2b3 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -13,7 +13,7 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION source-e2e-test -ENV APPLICATION_VERSION=1.0.0 +ENV APPLICATION_VERSION=1.0.1 ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index d87b2c73143e4..10abdd64d497b 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -153,6 +153,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.6.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | @@ -174,6 +175,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------| :--- | +| 0.2.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.2.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | BigQuery/BiqQuery denorm Destinations : Support dataset-id prefixed by project-id | | 0.2.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 133e3a310c0b3..24fe597d1e637 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -42,17 +42,11 @@ This mode throws an exception after receiving a configurable number of messages. ## CHANGELOG -### OSS (E2E Testing Destination) +The OSS and Cloud variants have the same version number starting from version `0.2.2`. | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--- | +| 0.2.2 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | | 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | | 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. | - -### Cloud (E2E Testing (`/dev/null`) Destination) - -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--- | -| 0.1.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | -| 0.1.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Create initial version. | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 1a4530c470e08..c1fd5e5efee01 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -217,6 +217,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | +| 0.4.7 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.4.6 | 2022-01-28 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.4.5 | 2021-12-29 | [#9184](https://github.com/airbytehq/airbyte/pull/9184) | Update connector fields title/description | | 0.4.4 | 2022-01-24 | [#9743](https://github.com/airbytehq/airbyte/pull/9743) | Fixed bug with dashes in schema name | diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index 8c9af97b3e418..f6222c647e02f 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -58,16 +58,11 @@ This mode is also excluded from the Cloud variant of this connector. ## Changelog -### OSS +The OSS and Cloud variants have the same version number. The Cloud variant was initially released with version `1.0.0`. | Version | Date | Pull request | Notes | | --- | --- | --- | --- | -| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. | +| 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | | 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | | 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. | - -### Cloud - -| Version | Date | Pull request | Notes | -| --- | --- | --- | --- | -| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | From 493d484e24c25e74a8200d2b837216a72b656e62 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 03:08:21 -0800 Subject: [PATCH 24/27] Add env to dockerfiles --- .../connectors/destination-dev-null/Dockerfile | 4 +++- .../connectors/source-e2e-test-cloud/Dockerfile | 4 +++- docs/integrations/sources/e2e-test.md | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile index 28a246b76686e..37af6bd7d3d34 100644 --- a/airbyte-integrations/connectors/destination-dev-null/Dockerfile +++ b/airbyte-integrations/connectors/destination-dev-null/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-dev-null +ENV APPLICATION_VERSION 0.2.2 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile index 57ae06e025587..949ed92b8f7f6 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile @@ -13,8 +13,10 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION source-e2e-test-cloud +ENV APPLICATION_VERSION 1.0.1 +ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=${APPLICATION_VERSION} LABEL io.airbyte.name=airbyte/source-e2e-test-cloud diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index f6222c647e02f..cc8b5a8bf9494 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -58,7 +58,7 @@ This mode is also excluded from the Cloud variant of this connector. ## Changelog -The OSS and Cloud variants have the same version number. The Cloud variant was initially released with version `1.0.0`. +The OSS and Cloud variants have the same version number. The Cloud variant was initially released at version `1.0.0`. | Version | Date | Pull request | Notes | | --- | --- | --- | --- | From d115b4b7b7ec75991d2ed6b97eb18cd82bfe835c Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 16:29:20 -0800 Subject: [PATCH 25/27] Fix e2e test connector dockerfil --- airbyte-integrations/connectors/destination-dev-null/Dockerfile | 2 +- airbyte-integrations/connectors/destination-e2e-test/Dockerfile | 2 +- .../connectors/source-e2e-test-cloud/Dockerfile | 2 +- airbyte-integrations/connectors/source-e2e-test/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile index 37af6bd7d3d34..63e26c67aea0e 100644 --- a/airbyte-integrations/connectors/destination-dev-null/Dockerfile +++ b/airbyte-integrations/connectors/destination-dev-null/Dockerfile @@ -18,5 +18,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index 86b16b0d7c8fc..42be5c9d6bd6b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -18,5 +18,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-e2e-test diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile index 949ed92b8f7f6..d5f623b0cd0b6 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile @@ -18,5 +18,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-e2e-test-cloud diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index 1f3509aeeb2b3..ddde252871e12 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -18,5 +18,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-e2e-test From d416f07c333d36f3f0fda2d6294f2db0ee58ecba Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 16:30:59 -0800 Subject: [PATCH 26/27] Fix snowflake bigquery dockerfile --- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- airbyte-integrations/connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index 524e7789a7bca..83c1e535a81a9 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -18,5 +18,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index b89d6cc69e9ab..8fe9d8d415a04 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -18,5 +18,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=0.6.6 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index a4626f14709c8..5d0c71a6fb6aa 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -21,5 +21,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV APPLICATION_VERSION 0.4.7 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=${APPLICATION_VERSION} +LABEL io.airbyte.version=0.4.7 LABEL io.airbyte.name=airbyte/destination-snowflake From d1c881a8eede1b9803c9cd7e26335d162ab2bf42 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sat, 29 Jan 2022 16:32:30 -0800 Subject: [PATCH 27/27] Mark new versions as unpublished --- docs/integrations/destinations/bigquery.md | 4 ++-- docs/integrations/destinations/e2e-test.md | 2 +- docs/integrations/destinations/snowflake.md | 2 +- docs/integrations/sources/e2e-test.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 10abdd64d497b..07c656ce5742a 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -153,7 +153,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| 0.6.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.6.6 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.6.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.6.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id | | 0.6.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | @@ -175,7 +175,7 @@ Therefore, Airbyte BigQuery destination will convert any invalid characters into | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------| :--- | -| 0.2.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.2.6 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | | 0.2.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | BigQuery/BiqQuery denorm Destinations : Support dataset-id prefixed by project-id | | 0.2.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 24fe597d1e637..7d0001553738f 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -46,7 +46,7 @@ The OSS and Cloud variants have the same version number starting from version `0 | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--- | -| 0.2.2 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.2.2 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | | 0.2.0 | 2021-12-16 | [\#8824](https://github.com/airbytehq/airbyte/pull/8824) | Add multiple logging modes. | | 0.1.0 | 2021-05-25 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) | Create initial version. | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index c1fd5e5efee01..d74da8ef38369 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -217,7 +217,7 @@ Finally, you need to add read/write permissions to your bucket with that email. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----- | :------ | -| 0.4.7 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 0.4.7 (unpublished) | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.4.6 | 2022-01-28 | [#9623](https://github.com/airbytehq/airbyte/pull/9623) | Add jdbc_url_params support for optional JDBC parameters | | 0.4.5 | 2021-12-29 | [#9184](https://github.com/airbytehq/airbyte/pull/9184) | Update connector fields title/description | | 0.4.4 | 2022-01-24 | [#9743](https://github.com/airbytehq/airbyte/pull/9743) | Fixed bug with dashes in schema name | diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index cc8b5a8bf9494..8d4616c359932 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -62,7 +62,7 @@ The OSS and Cloud variants have the same version number. The Cloud variant was i | Version | Date | Pull request | Notes | | --- | --- | --- | --- | -| 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 1.0.1 (unpublished) | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | | 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | | 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. |