Skip to content

Commit bd59726

Browse files
tulirenoctavia-squidington-iii
tuliren
and
octavia-squidington-iii
authored
Publish new connectors to log offending json fields in json to avro conversion for debugging (#12578)
* Publish new connectors to log debugging info in json to avro conversion * Add pull request id * auto-bump connector version * auto-bump connector version * auto-bump connector version * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 22b67d8 commit bd59726

File tree

11 files changed

+89
-72
lines changed

11 files changed

+89
-72
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
- name: BigQuery
2525
destinationDefinitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
2626
dockerRepository: airbyte/destination-bigquery
27-
dockerImageTag: 1.1.3
27+
dockerImageTag: 1.1.4
2828
documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
2929
icon: bigquery.svg
3030
resourceRequirements:
@@ -36,7 +36,7 @@
3636
- name: BigQuery (denormalized typed struct)
3737
destinationDefinitionId: 079d5540-f236-4294-ba7c-ade8fd918496
3838
dockerRepository: airbyte/destination-bigquery-denormalized
39-
dockerImageTag: 0.3.3
39+
dockerImageTag: 0.3.4
4040
documentationUrl: https://docs.airbyte.io/integrations/destinations/bigquery
4141
icon: bigquery.svg
4242
resourceRequirements:
@@ -83,7 +83,7 @@
8383
- name: Google Cloud Storage (GCS)
8484
destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
8585
dockerRepository: airbyte/destination-gcs
86-
dockerImageTag: 0.2.4
86+
dockerImageTag: 0.2.5
8787
documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
8888
icon: googlecloudstorage.svg
8989
resourceRequirements:
@@ -208,7 +208,7 @@
208208
- name: S3
209209
destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
210210
dockerRepository: airbyte/destination-s3
211-
dockerImageTag: 0.3.3
211+
dockerImageTag: 0.3.4
212212
documentationUrl: https://docs.airbyte.io/integrations/destinations/s3
213213
icon: s3.svg
214214
resourceRequirements:

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

+72-60
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@
285285
supported_destination_sync_modes:
286286
- "overwrite"
287287
- "append"
288-
- dockerImage: "airbyte/destination-bigquery:1.1.3"
288+
- dockerImage: "airbyte/destination-bigquery:1.1.4"
289289
spec:
290290
documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
291291
connectionSpecification:
@@ -494,7 +494,7 @@
494494
- "overwrite"
495495
- "append"
496496
- "append_dedup"
497-
- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.3"
497+
- dockerImage: "airbyte/destination-bigquery-denormalized:0.3.4"
498498
spec:
499499
documentationUrl: "https://docs.airbyte.io/integrations/destinations/bigquery"
500500
connectionSpecification:
@@ -1287,7 +1287,7 @@
12871287
- "overwrite"
12881288
- "append"
12891289
supportsNamespaces: true
1290-
- dockerImage: "airbyte/destination-gcs:0.2.4"
1290+
- dockerImage: "airbyte/destination-gcs:0.2.5"
12911291
spec:
12921292
documentationUrl: "https://docs.airbyte.io/integrations/destinations/gcs"
12931293
connectionSpecification:
@@ -3643,7 +3643,7 @@
36433643
supported_destination_sync_modes:
36443644
- "append"
36453645
- "overwrite"
3646-
- dockerImage: "airbyte/destination-s3:0.3.3"
3646+
- dockerImage: "airbyte/destination-s3:0.3.4"
36473647
spec:
36483648
documentationUrl: "https://docs.airbyte.io/integrations/destinations/s3"
36493649
connectionSpecification:
@@ -3657,40 +3657,46 @@
36573657
- "format"
36583658
additionalProperties: false
36593659
properties:
3660-
s3_endpoint:
3661-
title: "Endpoint"
3660+
access_key_id:
36623661
type: "string"
3663-
default: ""
3664-
description: "This is your S3 endpoint url.(if you are working with AWS\
3665-
\ S3, just leave empty)."
3662+
description: "The access key ID to access the S3 bucket. Airbyte requires\
3663+
\ Read and Write permissions to the given bucket. See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)\
3664+
\ on how to generate an access key."
3665+
title: "S3 Key ID *"
3666+
airbyte_secret: true
36663667
examples:
3667-
- "http://localhost:9000"
3668+
- "A012345678910EXAMPLE"
36683669
order: 0
3669-
s3_bucket_name:
3670-
title: "S3 Bucket Name"
3670+
secret_access_key:
36713671
type: "string"
3672-
description: "The name of the S3 bucket."
3672+
description: "The corresponding secret to the access key ID. See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys)"
3673+
title: "S3 Access Key *"
3674+
airbyte_secret: true
36733675
examples:
3674-
- "airbyte_sync"
3676+
- "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
36753677
order: 1
3676-
s3_bucket_path:
3677-
description: "Directory under the S3 bucket where data will be written."
3678+
s3_bucket_name:
3679+
title: "S3 Bucket Name *"
36783680
type: "string"
3681+
description: "The name of the S3 bucket. See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html)\
3682+
\ to create an S3 bucket."
36793683
examples:
3680-
- "data_sync/test"
3684+
- "airbyte_sync"
36813685
order: 2
3682-
s3_path_format:
3683-
description: "Format string on how data will be organized inside the S3\
3684-
\ bucket directory"
3686+
s3_bucket_path:
3687+
title: "S3 Bucket Path *"
3688+
description: "Directory under the S3 bucket where data will be written.\
3689+
\ See [this](https://docs.airbyte.com/integrations/destinations/s3#:~:text=to%20format%20the-,bucket%20path,-%3A)"
36853690
type: "string"
36863691
examples:
3687-
- "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
3692+
- "data_sync/test"
36883693
order: 3
36893694
s3_bucket_region:
3690-
title: "S3 Bucket Region"
3695+
title: "S3 Bucket Region *"
36913696
type: "string"
36923697
default: ""
3693-
description: "The region of the S3 bucket."
3698+
description: "The region of the S3 bucket. See [this](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions)\
3699+
\ for all region codes."
36943700
enum:
36953701
- ""
36963702
- "us-east-1"
@@ -3719,47 +3725,31 @@
37193725
- "us-gov-east-1"
37203726
- "us-gov-west-1"
37213727
order: 4
3722-
access_key_id:
3723-
type: "string"
3724-
description: "The access key id to access the S3 bucket. Airbyte requires\
3725-
\ Read and Write permissions to the given bucket, if not set, Airbyte\
3726-
\ will rely on Instance Profile."
3727-
title: "S3 Key Id"
3728-
airbyte_secret: true
3729-
examples:
3730-
- "A012345678910EXAMPLE"
3731-
order: 5
3732-
secret_access_key:
3733-
type: "string"
3734-
description: "The corresponding secret to the access key id, if S3 Key Id\
3735-
\ is set, then S3 Access Key must also be provided"
3736-
title: "S3 Access Key"
3737-
airbyte_secret: true
3738-
examples:
3739-
- "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"
3740-
order: 6
37413728
format:
3742-
title: "Output Format"
3729+
title: "Output Format *"
37433730
type: "object"
3744-
description: "Output data format"
3731+
description: "Format of the data output. See [this](https://docs.airbyte.com/integrations/destinations/s3/#output-schema)\
3732+
\ for more details"
37453733
oneOf:
37463734
- title: "Avro: Apache Avro"
37473735
required:
37483736
- "format_type"
37493737
- "compression_codec"
37503738
properties:
37513739
format_type:
3740+
title: "Format Type *"
37523741
type: "string"
37533742
enum:
37543743
- "Avro"
37553744
default: "Avro"
3745+
order: 0
37563746
compression_codec:
3757-
title: "Compression Codec"
3747+
title: "Compression Codec *"
37583748
description: "The compression algorithm used to compress data. Default\
37593749
\ to no compression."
37603750
type: "object"
37613751
oneOf:
3762-
- title: "no compression"
3752+
- title: "No Compression"
37633753
required:
37643754
- "codec"
37653755
properties:
@@ -3779,7 +3769,7 @@
37793769
- "Deflate"
37803770
default: "Deflate"
37813771
compression_level:
3782-
title: "Deflate level"
3772+
title: "Deflate Level"
37833773
description: "0: no compression & fastest, 9: best compression\
37843774
\ & slowest."
37853775
type: "integer"
@@ -3806,7 +3796,7 @@
38063796
- "xz"
38073797
default: "xz"
38083798
compression_level:
3809-
title: "Compression level"
3799+
title: "Compression Level"
38103800
description: "See <a href=\"https://commons.apache.org/proper/commons-compress/apidocs/org/apache/commons/compress/compressors/xz/XZCompressorOutputStream.html#XZCompressorOutputStream-java.io.OutputStream-int-\"\
38113801
>here</a> for details."
38123802
type: "integer"
@@ -3824,7 +3814,7 @@
38243814
- "zstandard"
38253815
default: "zstandard"
38263816
compression_level:
3827-
title: "Compression level"
3817+
title: "Compression Level"
38283818
description: "Negative levels are 'fast' modes akin to lz4 or\
38293819
\ snappy, levels above 9 are generally for archival purposes,\
38303820
\ and levels above 18 use a lot of memory."
@@ -3833,7 +3823,7 @@
38333823
minimum: -5
38343824
maximum: 22
38353825
include_checksum:
3836-
title: "Include checksum"
3826+
title: "Include Checksum"
38373827
description: "If true, include a checksum with each data block."
38383828
type: "boolean"
38393829
default: false
@@ -3846,8 +3836,9 @@
38463836
enum:
38473837
- "snappy"
38483838
default: "snappy"
3839+
order: 1
38493840
part_size_mb:
3850-
title: "Block Size (MB) for Amazon S3 multipart upload"
3841+
title: "Block Size (MB) for Amazon S3 multipart upload (Optional)"
38513842
description: "This is the size of a \"Part\" being buffered in memory.\
38523843
\ It limits the memory usage when writing. Larger values will allow\
38533844
\ to upload a bigger files and improve the speed, but consumes9\
@@ -3856,12 +3847,14 @@
38563847
default: 5
38573848
examples:
38583849
- 5
3850+
order: 2
38593851
- title: "CSV: Comma-Separated Values"
38603852
required:
38613853
- "format_type"
38623854
- "flattening"
38633855
properties:
38643856
format_type:
3857+
title: "Format Type *"
38653858
type: "string"
38663859
enum:
38673860
- "CSV"
@@ -3876,7 +3869,7 @@
38763869
- "No flattening"
38773870
- "Root level flattening"
38783871
part_size_mb:
3879-
title: "Block Size (MB) for Amazon S3 multipart upload"
3872+
title: "Block Size (MB) For Amazon S3 Multipart Upload (Optional)"
38803873
description: "This is the size of a \"Part\" being buffered in memory.\
38813874
\ It limits the memory usage when writing. Larger values will allow\
38823875
\ to upload a bigger files and improve the speed, but consumes9\
@@ -3910,17 +3903,18 @@
39103903
enum:
39113904
- "GZIP"
39123905
default: "GZIP"
3913-
- title: "JSON Lines: newline-delimited JSON"
3906+
- title: "JSON Lines: Newline-delimited JSON"
39143907
required:
39153908
- "format_type"
39163909
properties:
39173910
format_type:
3911+
title: "Format Type *"
39183912
type: "string"
39193913
enum:
39203914
- "JSONL"
39213915
default: "JSONL"
39223916
part_size_mb:
3923-
title: "Block Size (MB) for Amazon S3 multipart upload"
3917+
title: "Block Size (MB) For Amazon S3 Multipart Upload (Optional)"
39243918
description: "This is the size of a \"Part\" being buffered in memory.\
39253919
\ It limits the memory usage when writing. Larger values will allow\
39263920
\ to upload a bigger files and improve the speed, but consumes9\
@@ -3957,12 +3951,13 @@
39573951
- "format_type"
39583952
properties:
39593953
format_type:
3954+
title: "Format Type *"
39603955
type: "string"
39613956
enum:
39623957
- "Parquet"
39633958
default: "Parquet"
39643959
compression_codec:
3965-
title: "Compression Codec"
3960+
title: "Compression Codec (Optional)"
39663961
description: "The compression algorithm used to compress data pages."
39673962
type: "string"
39683963
enum:
@@ -3975,7 +3970,7 @@
39753970
- "ZSTD"
39763971
default: "UNCOMPRESSED"
39773972
block_size_mb:
3978-
title: "Block Size (Row Group Size) (MB)"
3973+
title: "Block Size (Row Group Size) (MB) (Optional)"
39793974
description: "This is the size of a row group being buffered in memory.\
39803975
\ It limits the memory usage when writing. Larger values will improve\
39813976
\ the IO when reading, but consume more memory when writing. Default:\
@@ -3985,15 +3980,15 @@
39853980
examples:
39863981
- 128
39873982
max_padding_size_mb:
3988-
title: "Max Padding Size (MB)"
3983+
title: "Max Padding Size (MB) (Optional)"
39893984
description: "Maximum size allowed as padding to align row groups.\
39903985
\ This is also the minimum size of a row group. Default: 8 MB."
39913986
type: "integer"
39923987
default: 8
39933988
examples:
39943989
- 8
39953990
page_size_kb:
3996-
title: "Page Size (KB)"
3991+
title: "Page Size (KB) (Optional)"
39973992
description: "The page size is for compression. A block is composed\
39983993
\ of pages. A page is the smallest unit that must be read fully\
39993994
\ to access a single record. If this value is too small, the compression\
@@ -4003,7 +3998,7 @@
40033998
examples:
40043999
- 1024
40054000
dictionary_page_size_kb:
4006-
title: "Dictionary Page Size (KB)"
4001+
title: "Dictionary Page Size (KB) (Optional)"
40074002
description: "There is one dictionary page per column per row group\
40084003
\ when dictionary encoding is used. The dictionary page size works\
40094004
\ like the page size but for dictionary. Default: 1024 KB."
@@ -4012,10 +4007,27 @@
40124007
examples:
40134008
- 1024
40144009
dictionary_encoding:
4015-
title: "Dictionary Encoding"
4010+
title: "Dictionary Encoding (Optional)"
40164011
description: "Default: true."
40174012
type: "boolean"
40184013
default: true
4014+
order: 5
4015+
s3_endpoint:
4016+
title: "Endpoint (Optional)"
4017+
type: "string"
4018+
default: ""
4019+
description: "This is your S3 endpoint url. (If you are working with AWS\
4020+
\ S3, you can leave blank). See [this](https://docs.aws.amazon.com/general/latest/gr/s3.html#:~:text=Service%20endpoints-,Amazon%20S3%20endpoints,-When%20you%20use)"
4021+
examples:
4022+
- "http://localhost:9000"
4023+
order: 6
4024+
s3_path_format:
4025+
title: "S3 Path Format (Optional)"
4026+
description: "Format string on how data will be organized inside the S3\
4027+
\ bucket directory. See [this](https://docs.airbyte.com/integrations/destinations/s3#:~:text=The%20full%20path%20of%20the%20output%20data%20with%20the%20default%20S3%20path%20format)"
4028+
type: "string"
4029+
examples:
4030+
- "${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
40194031
order: 7
40204032
supportsIncremental: true
40214033
supportsNormalization: false

‎airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
1717

1818
COPY --from=build /airbyte /airbyte
1919

20-
LABEL io.airbyte.version=0.3.3
20+
LABEL io.airbyte.version=0.3.4
2121
LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized

‎airbyte-integrations/connectors/destination-bigquery/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
1717

1818
COPY --from=build /airbyte /airbyte
1919

20-
LABEL io.airbyte.version=1.1.3
20+
LABEL io.airbyte.version=1.1.4
2121
LABEL io.airbyte.name=airbyte/destination-bigquery

‎airbyte-integrations/connectors/destination-databricks/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION destination-databricks
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=0.1.4
19+
LABEL io.airbyte.version=0.1.5
2020
LABEL io.airbyte.name=airbyte/destination-databricks

‎airbyte-integrations/connectors/destination-gcs/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=0.2.4
19+
LABEL io.airbyte.version=0.2.5
2020
LABEL io.airbyte.name=airbyte/destination-gcs

‎airbyte-integrations/connectors/destination-s3/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@ ENV APPLICATION destination-s3
1616

1717
COPY --from=build /airbyte /airbyte
1818

19-
LABEL io.airbyte.version=0.3.3
19+
LABEL io.airbyte.version=0.3.4
2020
LABEL io.airbyte.name=airbyte/destination-s3

0 commit comments

Comments
 (0)