Skip to content

Commit e17fa5b

Browse files
ycherniaievycherniaiev
authored
Update fields in source-connectors specifications: kustomer-singer, lemlist, linnworks, looker, marketo, mixpanel (#8578)
1 parent 13508dc commit e17fa5b

File tree

37 files changed

+153
-99
lines changed

37 files changed

+153
-99
lines changed

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c",
33
"name": "Looker",
44
"dockerRepository": "airbyte/source-looker",
5-
"dockerImageTag": "0.2.5",
5+
"dockerImageTag": "0.2.6",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/looker",
77
"icon": "looker.svg"
88
}

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a",
33
"name": "Mixpanel",
44
"dockerRepository": "airbyte/source-mixpanel",
5-
"dockerImageTag": "0.1.7",
5+
"dockerImageTag": "0.1.8.1",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel",
77
"icon": "mixpanel.svg"
88
}

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"sourceDefinitionId": "7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e",
33
"name": "Linnworks",
44
"dockerRepository": "airbyte/source-linnworks",
5-
"dockerImageTag": "0.1.0",
5+
"dockerImageTag": "0.1.5",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/linnworks",
77
"icon": "linworks.svg"
88
}

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9e0556f4-69df-4522-a3fb-03264d36b348.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348",
33
"name": "Marketo",
44
"dockerRepository": "airbyte/source-marketo",
5-
"dockerImageTag": "0.1.1",
5+
"dockerImageTag": "0.1.3",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/marketo",
77
"icon": "marketo.svg"
88
}

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/cd06e646-31bf-4dc8-af48-cbc6530fcad3.json

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"sourceDefinitionId": "cd06e646-31bf-4dc8-af48-cbc6530fcad3",
3+
"name": "Kustomer",
4+
"dockerRepository": "airbyte/source-kustomer-singer",
5+
"dockerImageTag": "0.1.2",
6+
"documentationUrl": "https://docs.airbyte.io/integrations/sources/kustomer"
7+
}

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

+10-4
Original file line numberDiff line numberDiff line change
@@ -363,14 +363,14 @@
363363
- name: Linnworks
364364
sourceDefinitionId: 7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e
365365
dockerRepository: airbyte/source-linnworks
366-
dockerImageTag: 0.1.4
366+
dockerImageTag: 0.1.5
367367
documentationUrl: https://docs.airbyte.io/integrations/sources/linnworks
368368
icon: linnworks.svg
369369
sourceType: api
370370
- name: Looker
371371
sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c
372372
dockerRepository: airbyte/source-looker
373-
dockerImageTag: 0.2.5
373+
dockerImageTag: 0.2.6
374374
documentationUrl: https://docs.airbyte.io/integrations/sources/looker
375375
icon: looker.svg
376376
sourceType: api
@@ -391,7 +391,7 @@
391391
- name: Marketo
392392
sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348
393393
dockerRepository: airbyte/source-marketo
394-
dockerImageTag: 0.1.2
394+
dockerImageTag: 0.1.3
395395
documentationUrl: https://docs.airbyte.io/integrations/sources/marketo
396396
icon: marketo.svg
397397
sourceType: api
@@ -412,7 +412,7 @@
412412
- name: Mixpanel
413413
sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a
414414
dockerRepository: airbyte/source-mixpanel
415-
dockerImageTag: 0.1.7
415+
dockerImageTag: 0.1.8.1
416416
documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel
417417
icon: mixpanel.svg
418418
sourceType: api
@@ -797,3 +797,9 @@
797797
documentationUrl: https://docs.airbyte.io/integrations/sources/zuora
798798
icon: zuora.svg
799799
sourceType: api
800+
- name: Kustomer
801+
sourceDefinitionId: cd06e646-31bf-4dc8-af48-cbc6530fcad3
802+
dockerRepository: airbyte/source-kustomer-singer
803+
dockerImageTag: 0.1.2
804+
documentationUrl: https://docs.airbyte.io/integrations/sources/kustomer
805+
sourceType: api

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

+52-20
Original file line numberDiff line numberDiff line change
@@ -3647,7 +3647,7 @@
36473647
- - "client_secret"
36483648
oauthFlowOutputParameters:
36493649
- - "refresh_token"
3650-
- dockerImage: "airbyte/source-linnworks:0.1.4"
3650+
- dockerImage: "airbyte/source-linnworks:0.1.5"
36513651
spec:
36523652
documentationUrl: "https://docs.airbyte.io/integrations/sources/linnworks"
36533653
connectionSpecification:
@@ -3662,23 +3662,27 @@
36623662
additionalProperties: false
36633663
properties:
36643664
application_id:
3665-
title: "Application ID"
3665+
title: "Application ID."
3666+
description: "Linnworks Application ID"
36663667
type: "string"
36673668
application_secret:
3668-
title: "Application secret"
3669+
title: "Application Secret"
3670+
description: "Linnworks Application Secret"
36693671
type: "string"
36703672
airbyte_secret: true
36713673
token:
3672-
title: "Token"
3674+
title: "API Token"
36733675
type: "string"
36743676
start_date:
3675-
title: "Start date"
3677+
title: "Start Date"
3678+
description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
3679+
\ data before this date will not be replicated."
36763680
type: "string"
36773681
format: "date-time"
36783682
supportsNormalization: false
36793683
supportsDBT: false
36803684
supported_destination_sync_modes: []
3681-
- dockerImage: "airbyte/source-looker:0.2.5"
3685+
- dockerImage: "airbyte/source-looker:0.2.6"
36823686
spec:
36833687
documentationUrl: "https://docs.airbyte.io/integrations/sources/looker"
36843688
connectionSpecification:
@@ -3693,6 +3697,7 @@
36933697
properties:
36943698
domain:
36953699
type: "string"
3700+
title: "Domain"
36963701
examples:
36973702
- "domainname.looker.com"
36983703
- "looker.clientname.com"
@@ -3777,7 +3782,7 @@
37773782
supportsNormalization: false
37783783
supportsDBT: false
37793784
supported_destination_sync_modes: []
3780-
- dockerImage: "airbyte/source-marketo:0.1.2"
3785+
- dockerImage: "airbyte/source-marketo:0.1.3"
37813786
spec:
37823787
documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo"
37833788
connectionSpecification:
@@ -3803,23 +3808,25 @@
38033808
client_id:
38043809
title: "Client ID"
38053810
type: "string"
3811+
description: "The Client ID of your Marketo developer application. See <a\
3812+
\ href=\"https://docs.airbyte.io/integrations/sources/marketo\"> the docs\
3813+
\ </a> for info on how to obtain this."
38063814
order: 0
3807-
description: "Your Marketo client_id. See <a href=\"https://docs.airbyte.io/integrations/sources/marketo\"\
3808-
> the docs </a> for info on how to obtain this."
38093815
airbyte_secret: true
38103816
client_secret:
38113817
title: "Client Secret"
38123818
type: "string"
3819+
description: "The Client Secret of your Marketo developer application. See\
3820+
\ <a href=\"https://docs.airbyte.io/integrations/sources/marketo\"> the\
3821+
\ docs </a> for info on how to obtain this."
38133822
order: 1
3814-
description: "Your Marketo client secret. See <a href=\"https://docs.airbyte.io/integrations/sources/marketo\"\
3815-
> the docs </a> for info on how to obtain this."
38163823
airbyte_secret: true
38173824
start_date:
38183825
title: "Start Date"
38193826
type: "string"
38203827
order: 2
3821-
description: "Data generated in Marketo after this date will be replicated.\
3822-
\ This date must be specified in the format YYYY-MM-DDT00:00:00Z."
3828+
description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
3829+
\ data before this date will not be replicated."
38233830
examples:
38243831
- "2020-09-25T00:00:00Z"
38253832
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
@@ -4176,7 +4183,7 @@
41764183
path_in_connector_config:
41774184
- "credentials"
41784185
- "client_secret"
4179-
- dockerImage: "airbyte/source-mixpanel:0.1.7"
4186+
- dockerImage: "airbyte/source-mixpanel:0.1.8.1"
41804187
spec:
41814188
documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel"
41824189
connectionSpecification:
@@ -4196,9 +4203,9 @@
41964203
attribution_window:
41974204
title: "Attribution Window"
41984205
type: "integer"
4199-
description: "Latency minimum number of days to look-back to account for\
4200-
\ delays in attributing accurate results. Default attribution window is\
4201-
\ 5 days."
4206+
description: " A period of time for attributing results to ads and the lookback\
4207+
\ period after those actions occur during which ad results are counted.Default\
4208+
\ attribution window is 5 days."
42024209
default: 5
42034210
project_timezone:
42044211
title: "Project Timezone"
@@ -4219,9 +4226,9 @@
42194226
start_date:
42204227
title: "Start Date"
42214228
type: "string"
4222-
description: "The default value to use if no bookmark exists for an endpoint.\
4223-
\ If this option is not set, the connector will replicate data from up\
4224-
\ to one year ago by default."
4229+
description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
4230+
\ data before this date will not be replicated. If this option is not\
4231+
\ set, the connector will replicate data from up to one year ago by default."
42254232
examples:
42264233
- "2021-11-16"
42274234
pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$"
@@ -7724,3 +7731,28 @@
77247731
supportsNormalization: false
77257732
supportsDBT: false
77267733
supported_destination_sync_modes: []
7734+
- dockerImage: "airbyte/source-kustomer-singer:0.1.2"
7735+
spec:
7736+
documentationUrl: "https://docs.airbyte.io/integrations/sources/kustomer"
7737+
connectionSpecification:
7738+
$schema: "http://json-schema.org/draft-07/schema#"
7739+
title: "Source Kustomer Singer Spec"
7740+
type: "object"
7741+
required:
7742+
- "api_token"
7743+
- "start_date"
7744+
additionalProperties: true
7745+
properties:
7746+
api_token:
7747+
type: "string"
7748+
description: "Kustomer API Token. See the <a href=\"https://developer.kustomer.com/kustomer-api-docs/reference/authentication\"\
7749+
>docs</a> on how to obtain this"
7750+
airbyte_secret: true
7751+
start_date:
7752+
type: "string"
7753+
description: "The date from which you'd like to replicate the data"
7754+
examples:
7755+
- "2019-01-01T00:00:00Z"
7756+
supportsNormalization: false
7757+
supportsDBT: false
7758+
supported_destination_sync_modes: []

‎airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ public class StagingFilenameGenerator {
2323
private int currentFileSuffixPartCount = 0;
2424

2525
// This variable is responsible to set the size of chunks size (In MB). After chunks created in
26-
// S3 or GCS they will be uploaded to Snowflake or Redshift. These service have some limitations for the uploading file.
26+
// S3 or GCS they will be uploaded to Snowflake or Redshift. These service have some limitations for
27+
// the uploading file.
2728
// So we make the calculation to determine how many parts we can put to the single chunk file.
2829
private final long iterations;
2930

‎airbyte-integrations/connectors/destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java

+10-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,20 @@
1+
/*
2+
* Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
*/
4+
15
package io.airbyte.integrations.destination.jdbc.constants;
26

37
import io.aesy.datasize.ByteUnit.IEC;
48
import io.aesy.datasize.DataSize;
59

610
public interface GlobalDataSizeConstants {
11+
712
/** 256 MB to BYTES as comparison will be done in BYTES */
813
int DEFAULT_MAX_BATCH_SIZE_BYTES = DataSize.of(256L, IEC.MEBIBYTE).toUnit(IEC.BYTE).getValue().intValue();
9-
/** This constant determines the max possible size of file(e.g. 1 GB / 256 megabytes ≈ 4 chunks of file)
10-
see StagingFilenameGenerator.java:28
11-
*/
14+
/**
15+
* This constant determines the max possible size of file(e.g. 1 GB / 256 megabytes ≈ 4 chunks of
16+
* file) see StagingFilenameGenerator.java:28
17+
*/
1218
long MAX_FILE_SIZE = DataSize.of(1L, IEC.GIBIBYTE).toUnit(IEC.BYTE).getValue().longValue();
19+
1320
}

‎airbyte-integrations/connectors/destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGeneratorTest.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,15 @@
66

77
import static org.junit.jupiter.api.Assertions.*;
88

9-
import io.aesy.datasize.ByteUnit.IEC;
10-
import io.aesy.datasize.DataSize;
119
import io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants;
1210
import org.apache.commons.lang3.RandomStringUtils;
1311
import org.junit.jupiter.api.Test;
1412

1513
class StagingFilenameGeneratorTest {
1614

1715
private static final String STREAM_NAME = RandomStringUtils.randomAlphabetic(5).toLowerCase();
18-
// Equal to GlobalDataSizeConstants.MAX_BYTE_PARTS_PER_FILE / GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES
16+
// Equal to GlobalDataSizeConstants.MAX_BYTE_PARTS_PER_FILE /
17+
// GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES
1918
// because <insert explanation here>
2019
private static final int EXPECTED_ITERATIONS_WITH_STANDARD_BYTE_BUFFER = 4;
2120
private static final StagingFilenameGenerator FILENAME_GENERATOR =

‎airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public class SnowflakeDatabase {
2222
private static final Duration NETWORK_TIMEOUT = Duration.ofMinutes(1);
2323
private static final Duration QUERY_TIMEOUT = Duration.ofHours(3);
2424
private static final SnowflakeSQLNameTransformer nameTransformer = new SnowflakeSQLNameTransformer();
25+
2526
public static Connection getConnection(final JsonNode config) throws SQLException {
2627
final String connectUrl = String.format("jdbc:snowflake://%s", config.get("host").asText());
2728

‎airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java

-2
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
package io.airbyte.integrations.destination.snowflake;
66

77
import com.google.cloud.storage.Storage;
8-
import io.aesy.datasize.ByteUnit.IEC;
9-
import io.aesy.datasize.DataSize;
108
import io.airbyte.db.jdbc.JdbcDatabase;
119
import io.airbyte.integrations.destination.ExtendedNameTransformer;
1210
import io.airbyte.integrations.destination.jdbc.SqlOperations;

‎airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java

-2
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
import io.airbyte.integrations.destination.s3.S3DestinationConfig;
1313
import io.airbyte.protocol.models.DestinationSyncMode;
1414
import java.sql.SQLException;
15-
import org.slf4j.Logger;
16-
import org.slf4j.LoggerFactory;
1715

1816
public class SnowflakeS3StreamCopier extends LegacyS3StreamCopier {
1917

‎airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,7 @@
174174
"part_size": {
175175
"type": "integer",
176176
"default": 5,
177-
"examples": [
178-
5
179-
],
177+
"examples": [5],
180178
"description": "Optional. Increase this if syncing tables larger than 100GB. Only relevant for COPY. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default limit of 100GB tables. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care.",
181179
"title": "Stream Part Size",
182180
"order": 5

‎airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java

+9-8
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44

55
package io.airbyte.integrations.destination.snowflake;
66

7+
import static org.junit.jupiter.api.Assertions.*;
8+
79
import com.fasterxml.jackson.databind.JsonNode;
810
import com.fasterxml.jackson.databind.node.ObjectNode;
911
import io.airbyte.commons.json.Jsons;
1012
import io.airbyte.commons.string.Strings;
1113
import io.airbyte.protocol.models.AirbyteConnectionStatus;
12-
import org.junit.jupiter.api.Test;
13-
1414
import java.io.IOException;
1515
import java.nio.file.Files;
1616
import java.nio.file.Paths;
@@ -19,10 +19,10 @@
1919
import java.sql.SQLException;
2020
import java.sql.Statement;
2121
import java.util.Properties;
22-
23-
import static org.junit.jupiter.api.Assertions.*;
22+
import org.junit.jupiter.api.Test;
2423

2524
class SnowflakeDestinationIntegrationTest {
25+
2626
private final SnowflakeSQLNameTransformer namingResolver = new SnowflakeSQLNameTransformer();
2727

2828
@Test
@@ -46,11 +46,11 @@ public void testInvalidSchemaName() {
4646
public void syncWithNamingResolver() throws IOException, SQLException {
4747
final JsonNode config = getConfig();
4848
final String createSchemaQuery = String.format("CREATE SCHEMA %s", namingResolver.getIdentifier(config.get("schema").asText()));
49-
Connection connection =null;
49+
Connection connection = null;
5050
try {
5151
connection = SnowflakeDatabase.getConnection(config);
5252
connection.createStatement().execute(createSchemaQuery);
53-
}finally {
53+
} finally {
5454
if (connection != null) {
5555
final String dropSchemaQuery = String.format("DROP SCHEMA IF EXISTS %s", namingResolver.getIdentifier(config.get("schema").asText()));
5656
connection.createStatement().execute(dropSchemaQuery);
@@ -82,8 +82,8 @@ public Connection getConnection(JsonNode config, boolean useNameTransformer) thr
8282
properties.put("database", config.get("database").asText());
8383
properties.put("role", config.get("role").asText());
8484
properties.put("schema", useNameTransformer
85-
? namingResolver.getIdentifier(config.get("schema").asText())
86-
: config.get("schema").asText());
85+
? namingResolver.getIdentifier(config.get("schema").asText())
86+
: config.get("schema").asText());
8787

8888
properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON");
8989

@@ -96,4 +96,5 @@ private JsonNode getConfig() throws IOException {
9696
((ObjectNode) config).put("schema", schemaName);
9797
return config;
9898
}
99+
99100
}

0 commit comments

Comments
 (0)