Skip to content

Commit 34d43cc

Browse files
davydov-dlazebnyi
davydov-d
and
lazebnyi
authored
Source Marketo: set high test strictness level (#21790)
* #19055 source marketo: set high test strictness level * #19055 source marketo: increase incremental test timeout * #19055 source Marketo: fix stream schemas * #19055 source marketoL fix unit test * Updated expected records * Updated expected records Co-authored-by: Serhii Lazebnyi <[email protected]>
1 parent b0a81ba commit 34d43cc

File tree

9 files changed

+313
-165
lines changed

9 files changed

+313
-165
lines changed

‎airbyte-integrations/connectors/source-marketo/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo
3434
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
3535
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
3636

37-
LABEL io.airbyte.version=0.1.11
37+
LABEL io.airbyte.version=1.0.0
3838
LABEL io.airbyte.name=airbyte/source-marketo

‎airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml

Lines changed: 73 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,82 @@
11
# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
22
# for more information about how to configure these tests
33
connector_image: airbyte/source-marketo:dev
4-
tests:
4+
test_strictness_level: high
5+
acceptance_tests:
56
spec:
6-
- spec_path: "source_marketo/spec.json"
7+
tests:
8+
- spec_path: "source_marketo/spec.json"
79
connection:
8-
- config_path: "secrets/config.json"
9-
status: "succeed"
10-
- config_path: "integration_tests/invalid_config.json"
11-
status: "failed"
10+
tests:
11+
- config_path: "secrets/config.json"
12+
status: "succeed"
13+
- config_path: "integration_tests/invalid_config.json"
14+
status: "failed"
1215
discovery:
13-
- config_path: "secrets/config.json"
16+
tests:
17+
- config_path: "secrets/config.json"
1418
basic_read:
15-
- config_path: "secrets/config.json"
16-
configured_catalog_path: "integration_tests/configured_catalog.json"
17-
empty_streams: ["lists", "campaigns"]
18-
timeout_seconds: 3600
19-
expect_records:
20-
path: "integration_tests/expected_records.jsonl"
19+
tests:
20+
- config_path: "secrets/config.json"
21+
expect_records:
22+
path: "integration_tests/expected_records.jsonl"
23+
empty_streams:
24+
- name: "activities_update_opportunity"
25+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
26+
- name: "activities_change_segment"
27+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
28+
- name: "activities_document_interacted_with"
29+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
30+
- name: "activities_call_webhook"
31+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
32+
- name: "activities_push_leadto_marketo"
33+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
34+
- name: "activities_change_revenue_stage"
35+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
36+
- name: "activities_dialogue_goal_reached"
37+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
38+
- name: "activities_removefrom_opportunity"
39+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
40+
- name: "activities_received_forwardto_friend_email"
41+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
42+
- name: "activities_open_sales_email"
43+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
44+
- name: "activities_share_content"
45+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
46+
- name: "activities_sales_email_bounced"
47+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
48+
- name: "activities_dialogue_engaged"
49+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
50+
- name: "activities_dialogue_appointment_scheduled"
51+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
52+
- name: "activities_click_sales_email"
53+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
54+
- name: "activities_sent_forwardto_friend_email"
55+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
56+
- name: "activities_click_link"
57+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
58+
- name: "activities_send_sales_email"
59+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
60+
- name: "activities_change_revenue_stage_manually"
61+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
62+
- name: "activities_receive_sales_email"
63+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
64+
- name: "activities_replyto_sales_email"
65+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
66+
- name: "activities_addto_opportunity"
67+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
68+
- name: "activities_open_email"
69+
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
70+
# 52 streams, most of them use BULK API therefore it takes much time to run a sync
71+
timeout_seconds: 9000
2172
incremental:
22-
- config_path: "secrets/config.json"
23-
configured_catalog_path: "integration_tests/incremental_catalog.json"
24-
future_state_path: "integration_tests/abnormal_state.json"
25-
timeout_seconds: 3600
73+
tests:
74+
- config_path: "secrets/config.json"
75+
configured_catalog_path: "integration_tests/incremental_catalog.json"
76+
future_state:
77+
future_state_path: "integration_tests/abnormal_state.json"
78+
timeout_seconds: 3600
2679
full_refresh:
27-
- config_path: "secrets/config.json"
28-
configured_catalog_path: "integration_tests/configured_catalog.json"
29-
timeout_seconds: 3600
80+
tests:
81+
- config_path: "secrets/config.json"
82+
configured_catalog_path: "integration_tests/configured_catalog.json"

‎airbyte-integrations/connectors/source-marketo/integration_tests/catalog.json

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎airbyte-integrations/connectors/source-marketo/integration_tests/configured_catalog.json

Lines changed: 42 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,65 +4,84 @@
44
"stream": {
55
"name": "programs",
66
"json_schema": {},
7-
"supported_sync_modes": ["incremental"],
7+
"supported_sync_modes": ["full_refresh", "incremental"],
88
"source_defined_cursor": true,
9-
"default_cursor_field": []
9+
"default_cursor_field": ["updatedAt"],
10+
"source_defined_primary_key": [["id"]]
1011
},
11-
"sync_mode": "incremental",
12-
"destination_sync_mode": "append"
12+
"sync_mode": "full_refresh",
13+
"destination_sync_mode": "append",
14+
"cursor_field": ["updatedAt"],
15+
"primary_key": [["id"]]
1316
},
1417
{
1518
"stream": {
1619
"name": "campaigns",
1720
"json_schema": {},
18-
"supported_sync_modes": ["incremental"],
21+
"supported_sync_modes": ["full_refresh", "incremental"],
1922
"source_defined_cursor": true,
20-
"default_cursor_field": []
23+
"default_cursor_field": ["createdAt"],
24+
"source_defined_primary_key": [["id"]]
2125
},
22-
"sync_mode": "incremental",
23-
"destination_sync_mode": "append"
26+
"sync_mode": "full_refresh",
27+
"destination_sync_mode": "append",
28+
"cursor_field": ["createdAt"],
29+
"primary_key": [["id"]]
2430
},
2531
{
2632
"stream": {
2733
"name": "lists",
2834
"json_schema": {},
29-
"supported_sync_modes": ["incremental"],
35+
"supported_sync_modes": ["full_refresh", "incremental"],
3036
"source_defined_cursor": true,
31-
"default_cursor_field": []
37+
"default_cursor_field": ["createdAt"],
38+
"source_defined_primary_key": [["id"]]
3239
},
33-
"sync_mode": "incremental",
34-
"destination_sync_mode": "append"
40+
"sync_mode": "full_refresh",
41+
"destination_sync_mode": "append",
42+
"cursor_field": ["createdAt"],
43+
"primary_key": [["id"]]
3544
},
3645
{
3746
"stream": {
3847
"name": "leads",
3948
"json_schema": {},
40-
"supported_sync_modes": ["incremental"],
41-
"source_defined_cursor": true,
42-
"default_cursor_field": []
49+
"supported_sync_modes": ["full_refresh", "incremental"],
50+
"default_cursor_field": ["updatedAt"],
51+
"source_defined_primary_key": [["id"]]
4352
},
44-
"sync_mode": "incremental",
45-
"destination_sync_mode": "append"
53+
"sync_mode": "full_refresh",
54+
"destination_sync_mode": "append",
55+
"cursor_field": ["updatedAt"],
56+
"primary_key": [["id"]]
4657
},
4758
{
4859
"stream": {
4960
"name": "activity_types",
5061
"json_schema": {},
51-
"supported_sync_modes": ["full_refresh"]
62+
"supported_sync_modes": ["full_refresh"],
63+
"source_defined_cursor": false,
64+
"default_cursor_field": null,
65+
"source_defined_primary_key": [["id"]]
5266
},
5367
"sync_mode": "full_refresh",
54-
"destination_sync_mode": "append"
68+
"destination_sync_mode": "append",
69+
"cursor_field": null,
70+
"primary_key": [["id"]]
5571
},
5672
{
5773
"stream": {
5874
"name": "activities_visit_webpage",
5975
"json_schema": {},
60-
"supported_sync_modes": ["incremental"],
76+
"supported_sync_modes": ["full_refresh", "incremental"],
6177
"source_defined_cursor": true,
62-
"default_cursor_field": []
78+
"default_cursor_field": ["activityDate"],
79+
"source_defined_primary_key": [["marketoGUID"]]
6380
},
64-
"sync_mode": "incremental",
65-
"destination_sync_mode": "append"
81+
"sync_mode": "full_refresh",
82+
"destination_sync_mode": "append",
83+
"cursor_field": ["activityDate"],
84+
"primary_key": [["marketoGUID"]]
6685
}
6786
]
6887
}

0 commit comments

Comments
 (0)