Skip to content

Publish new connectors to log offending json fields in json to avro conversion for debugging #12578

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
May 4, 2022
Merged

Publish new connectors to log offending json fields in json to avro conversion for debugging #12578

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
920f4e7
Publish new connectors to log debugging info in json to avro conversion
tuliren May 4, 2022
532cfaf
Add pull request id
tuliren May 4, 2022
5361476
auto-bump connector version
octavia-squidington-iii May 4, 2022
0258964
auto-bump connector version
octavia-squidington-iii May 4, 2022
fe9360c
Merge branch 'liren/publish-json-avro-logging' of https://github.com/…
octavia-squidington-iii May 4, 2022
714e876
auto-bump connector version
octavia-squidington-iii May 4, 2022
6b53ff0
Merge branch 'liren/publish-json-avro-logging' of https://github.com/…
octavia-squidington-iii May 4, 2022
3ee8899
auto-bump connector version
octavia-squidington-iii May 4, 2022
153c857
Merge branch 'liren/publish-json-avro-logging' of https://github.com/…
octavia-squidington-iii May 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions airbyte-config/init/src/main/resources/seed/destination_definitions.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
- name: BigQuery
destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
dockerRepository: airbyte/destination-bigquery
dockerImageTag: 1.1.3
dockerImageTag: 1.1.4
documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
icon: bigquery.svg
resourceRequirements:
Expand All @@ -36,7 +36,7 @@
- name: BigQuery (denormalized typed struct)
destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496
dockerRepository: airbyte/destination-bigquery-denormalized
dockerImageTag: 0.3.3
dockerImageTag: 0.3.4
documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
icon: bigquery.svg
resourceRequirements:
Expand Down Expand Up @@ -83,7 +83,7 @@
- name: Google Cloud Storage (GCS)
destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
dockerRepository: airbyte/destination-gcs
dockerImageTag: 0.2.4
dockerImageTag: 0.2.5
documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
icon: googlecloudstorage.svg
resourceRequirements:
Expand Down Expand Up @@ -208,7 +208,7 @@
- name: S3
destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
dockerRepository: airbyte/destination-s3
dockerImageTag: 0.3.3
dockerImageTag: 0.3.4
documentationUrl: https://docs.airbyte.io/integrations/destinations/s3
icon: s3.svg
resourceRequirements:
Expand Down
132 changes: 72 additions & 60 deletions airbyte-config/init/src/main/resources/seed/destination_specs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@
supported_destination_sync_modes:
- "overwrite"
- "append"
- dockerImage: "airbyte/destination-bigquery:1.1.3"
- dockerImage: "airbyte/destination-bigquery:1.1.4"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
connectionSpecification:
Expand Down Expand Up @@ -494,7 +494,7 @@
- "overwrite"
- "append"
- "append_dedup"
- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.3"
- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.4"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
connectionSpecification:
Expand Down Expand Up @@ -1287,7 +1287,7 @@
- "overwrite"
- "append"
supportsNamespaces: true
- dockerImage: "airbyte/destination-gcs:0.2.4"
- dockerImage: "airbyte/destination-gcs:0.2.5"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs"
connectionSpecification:
Expand Down Expand Up @@ -3643,7 +3643,7 @@
supported_destination_sync_modes:
- "append"
- "overwrite"
- dockerImage: "airbyte/destination-s3:0.3.3"
- dockerImage: "airbyte/destination-s3:0.3.4"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3"
connectionSpecification:
Expand All @@ -3657,40 +3657,46 @@
- "format"
additionalProperties: false
properties:
s3_endpoint:
title: "Endpoint"
access_key_id:
type: "string"
default: ""
description: "This is your S3 endpoint url.(if you are working with AWS\
\ S3, just leave empty)."
description: "The access key ID to access the S3 bucket. Airbyte requires\
\ Read and Write permissions to the given bucket. See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)\
\ on how to generate an access key."
title: "S3 Key ID *"
airbyte_secret: true
examples:
- "http://localhost:9000"
- "A012345678910EXAMPLE"
order: 0
s3_bucket_name:
title: "S3 Bucket Name"
secret_access_key:
type: "string"
description: "The name of the S3 bucket."
description: "The corresponding secret to the access key ID. See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)"
title: "S3 Access Key *"
airbyte_secret: true
examples:
- "airbyte_sync"
- "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
order: 1
s3_bucket_path:
description: "Directory under the S3 bucket where data will be written."
s3_bucket_name:
title: "S3 Bucket Name *"
type: "string"
description: "The name of the S3 bucket. See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)\
\ to create an S3 bucket."
examples:
- "data_sync/test"
- "airbyte_sync"
order: 2
s3_path_format:
description: "Format string on how data will be organized inside the S3\
\ bucket directory"
s3_bucket_path:
title: "S3 Bucket Path *"
description: "Directory under the S3 bucket where data will be written.\
\ See [this](https://docs.airbyte.com/integrations/destinations/s3#:~:text=to%20format%20the-,bucket%20path,-%3A)"
type: "string"
examples:
- "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
- "data_sync/test"
order: 3
s3_bucket_region:
title: "S3 Bucket Region"
title: "S3 Bucket Region *"
type: "string"
default: ""
description: "The region of the S3 bucket."
description: "The region of the S3 bucket. See [this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions)\
\ for all region codes."
enum:
- ""
- "us-east-1"
Expand Down Expand Up @@ -3719,47 +3725,31 @@
- "us-gov-east-1"
- "us-gov-west-1"
order: 4
access_key_id:
type: "string"
description: "The access key id to access the S3 bucket. Airbyte requires\
\ Read and Write permissions to the given bucket, if not set, Airbyte\
\ will rely on Instance Profile."
title: "S3 Key Id"
airbyte_secret: true
examples:
- "A012345678910EXAMPLE"
order: 5
secret_access_key:
type: "string"
description: "The corresponding secret to the access key id, if S3 Key Id\
\ is set, then S3 Access Key must also be provided"
title: "S3 Access Key"
airbyte_secret: true
examples:
- "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
order: 6
format:
title: "Output Format"
title: "Output Format *"
type: "object"
description: "Output data format"
description: "Format of the data output. See [this](https://docs.airbyte.com/integrations/destinations/s3/#output-schema)\
\ for more details"
oneOf:
- title: "Avro: Apache Avro"
required:
- "format_type"
- "compression_codec"
properties:
format_type:
title: "Format Type *"
type: "string"
enum:
- "Avro"
default: "Avro"
order: 0
compression_codec:
title: "Compression Codec"
title: "Compression Codec *"
description: "The compression algorithm used to compress data. Default\
\ to no compression."
type: "object"
oneOf:
- title: "no compression"
- title: "No Compression"
required:
- "codec"
properties:
Expand All @@ -3779,7 +3769,7 @@
- "Deflate"
default: "Deflate"
compression_level:
title: "Deflate level"
title: "Deflate Level"
description: "0: no compression & fastest, 9: best compression\
\ & slowest."
type: "integer"
Expand All @@ -3806,7 +3796,7 @@
- "xz"
default: "xz"
compression_level:
title: "Compression level"
title: "Compression Level"
description: "See <a href=\"https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/xz/XZCompressorOutputStream.html#XZCompressorOutputStream-java.io.OutputStream-int-\"\
>here</a> for details."
type: "integer"
Expand All @@ -3824,7 +3814,7 @@
- "zstandard"
default: "zstandard"
compression_level:
title: "Compression level"
title: "Compression Level"
description: "Negative levels are 'fast' modes akin to lz4 or\
\ snappy, levels above 9 are generally for archival purposes,\
\ and levels above 18 use a lot of memory."
Expand All @@ -3833,7 +3823,7 @@
minimum: -5
maximum: 22
include_checksum:
title: "Include checksum"
title: "Include Checksum"
description: "If true, include a checksum with each data block."
type: "boolean"
default: false
Expand All @@ -3846,8 +3836,9 @@
enum:
- "snappy"
default: "snappy"
order: 1
part_size_mb:
title: "Block Size (MB) for Amazon S3 multipart upload"
title: "Block Size (MB) for Amazon S3 multipart upload (Optional)"
description: "This is the size of a \"Part\" being buffered in memory.\
\ It limits the memory usage when writing. Larger values will allow\
\ to upload a bigger files and improve the speed, but consumes9\
Expand All @@ -3856,12 +3847,14 @@
default: 5
examples:
- 5
order: 2
- title: "CSV: Comma-Separated Values"
required:
- "format_type"
- "flattening"
properties:
format_type:
title: "Format Type *"
type: "string"
enum:
- "CSV"
Expand All @@ -3876,7 +3869,7 @@
- "No flattening"
- "Root level flattening"
part_size_mb:
title: "Block Size (MB) for Amazon S3 multipart upload"
title: "Block Size (MB) For Amazon S3 Multipart Upload (Optional)"
description: "This is the size of a \"Part\" being buffered in memory.\
\ It limits the memory usage when writing. Larger values will allow\
\ to upload a bigger files and improve the speed, but consumes9\
Expand Down Expand Up @@ -3910,17 +3903,18 @@
enum:
- "GZIP"
default: "GZIP"
- title: "JSON Lines: newline-delimited JSON"
- title: "JSON Lines: Newline-delimited JSON"
required:
- "format_type"
properties:
format_type:
title: "Format Type *"
type: "string"
enum:
- "JSONL"
default: "JSONL"
part_size_mb:
title: "Block Size (MB) for Amazon S3 multipart upload"
title: "Block Size (MB) For Amazon S3 Multipart Upload (Optional)"
description: "This is the size of a \"Part\" being buffered in memory.\
\ It limits the memory usage when writing. Larger values will allow\
\ to upload a bigger files and improve the speed, but consumes9\
Expand Down Expand Up @@ -3957,12 +3951,13 @@
- "format_type"
properties:
format_type:
title: "Format Type *"
type: "string"
enum:
- "Parquet"
default: "Parquet"
compression_codec:
title: "Compression Codec"
title: "Compression Codec (Optional)"
description: "The compression algorithm used to compress data pages."
type: "string"
enum:
Expand All @@ -3975,7 +3970,7 @@
- "ZSTD"
default: "UNCOMPRESSED"
block_size_mb:
title: "Block Size (Row Group Size) (MB)"
title: "Block Size (Row Group Size) (MB) (Optional)"
description: "This is the size of a row group being buffered in memory.\
\ It limits the memory usage when writing. Larger values will improve\
\ the IO when reading, but consume more memory when writing. Default:\
Expand All @@ -3985,15 +3980,15 @@
examples:
- 128
max_padding_size_mb:
title: "Max Padding Size (MB)"
title: "Max Padding Size (MB) (Optional)"
description: "Maximum size allowed as padding to align row groups.\
\ This is also the minimum size of a row group. Default: 8 MB."
type: "integer"
default: 8
examples:
- 8
page_size_kb:
title: "Page Size (KB)"
title: "Page Size (KB) (Optional)"
description: "The page size is for compression. A block is composed\
\ of pages. A page is the smallest unit that must be read fully\
\ to access a single record. If this value is too small, the compression\
Expand All @@ -4003,7 +3998,7 @@
examples:
- 1024
dictionary_page_size_kb:
title: "Dictionary Page Size (KB)"
title: "Dictionary Page Size (KB) (Optional)"
description: "There is one dictionary page per column per row group\
\ when dictionary encoding is used. The dictionary page size works\
\ like the page size but for dictionary. Default: 1024 KB."
Expand All @@ -4012,10 +4007,27 @@
examples:
- 1024
dictionary_encoding:
title: "Dictionary Encoding"
title: "Dictionary Encoding (Optional)"
description: "Default: true."
type: "boolean"
default: true
order: 5
s3_endpoint:
title: "Endpoint (Optional)"
type: "string"
default: ""
description: "This is your S3 endpoint url. (If you are working with AWS\
\ S3, you can leave blank). See [this](https://docs.aws.amazon.com/general/latest/gr/s3.html#:~:text=Service%20endpoints-,Amazon%20S3%20endpoints,-When%20you%20use)"
examples:
- "http://localhost:9000"
order: 6
s3_path_format:
title: "S3 Path Format (Optional)"
description: "Format string on how data will be organized inside the S3\
\ bucket directory. See [this](https://docs.airbyte.com/integrations/destinations/s3#:~:text=The%20full%20path%20of%20the%20output%20data%20with%20the%20default%20S3%20path%20format)"
type: "string"
examples:
- "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
order: 7
supportsIncremental: true
supportsNormalization: false
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.3.3
LABEL io.airbyte.version=0.3.4
LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-bigquery/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=1.1.3
LABEL io.airbyte.version=1.1.4
LABEL io.airbyte.name=airbyte/destination-bigquery
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-databricks/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION destination-databricks

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.1.4
LABEL io.airbyte.version=0.1.5
LABEL io.airbyte.name=airbyte/destination-databricks
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-gcs/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.2.4
LABEL io.airbyte.version=0.2.5
LABEL io.airbyte.name=airbyte/destination-gcs
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/destination-s3/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ ENV APPLICATION destination-s3

COPY --from=build /airbyte /airbyte

LABEL io.airbyte.version=0.3.3
LABEL io.airbyte.version=0.3.4
LABEL io.airbyte.name=airbyte/destination-s3
Loading