Skip to content

Commit b16590e

Browse files
pnilanpnilan
authored
✨ Source Harvest: Migrate to Low Code (#35863)
1 parent 714eea3 commit b16590e

40 files changed

+3090
-932
lines changed

‎airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml

+5-9
Original file line numberDiff line numberDiff line change
@@ -3,34 +3,30 @@ test_strictness_level: "high"
33
acceptance_tests:
44
spec:
55
tests:
6-
- spec_path: "source_harvest/spec.json"
6+
- spec_path: "source_harvest/spec.yaml"
77
connection:
88
tests:
9-
- config_path: "secrets/config.json"
10-
status: "succeed"
11-
- config_path: "secrets/old_config.json"
12-
status: "succeed"
139
- config_path: "secrets/config_oauth.json"
1410
status: "succeed"
1511
- config_path: "integration_tests/invalid_config.json"
1612
status: "failed"
1713
discovery:
1814
tests:
19-
- config_path: "secrets/config.json"
15+
- config_path: "secrets/config_oauth.json"
2016
basic_read:
2117
tests:
22-
- config_path: "secrets/config.json"
18+
- config_path: "secrets/config_oauth.json"
2319
expect_records:
2420
path: "integration_tests/expected_records.jsonl"
2521
fail_on_extra_columns: false
2622
incremental:
2723
tests:
28-
- config_path: "secrets/config_with_date_range.json"
24+
- config_path: "secrets/config_oauth.json"
2925
configured_catalog_path: "integration_tests/incremental_catalog.json"
3026
future_state:
3127
future_state_path: "integration_tests/abnormal_state.json"
3228
timeout_seconds: 2400
3329
full_refresh:
3430
tests:
35-
- config_path: "secrets/config.json"
31+
- config_path: "secrets/config_oauth.json"
3632
configured_catalog_path: "integration_tests/configured_catalog.json"

‎airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json

+76-28
Original file line numberDiff line numberDiff line change
@@ -20,20 +20,6 @@
2020
"stream_descriptor": { "name": "invoices" }
2121
}
2222
},
23-
{
24-
"type": "STREAM",
25-
"stream": {
26-
"stream_state": { "updated_at": "2217-06-26T21:20:07Z" },
27-
"stream_descriptor": { "name": "invoice_messages" }
28-
}
29-
},
30-
{
31-
"type": "STREAM",
32-
"stream": {
33-
"stream_state": { "updated_at": "2217-06-26T21:20:07Z" },
34-
"stream_descriptor": { "name": "invoice_payments" }
35-
}
36-
},
3723
{
3824
"type": "STREAM",
3925
"stream": {
@@ -48,13 +34,6 @@
4834
"stream_descriptor": { "name": "estimates" }
4935
}
5036
},
51-
{
52-
"type": "STREAM",
53-
"stream": {
54-
"stream_state": { "updated_at": "2217-06-26T21:20:07Z" },
55-
"stream_descriptor": { "name": "estimate_messages" }
56-
}
57-
},
5837
{
5938
"type": "STREAM",
6039
"stream": {
@@ -125,13 +104,6 @@
125104
"stream_descriptor": { "name": "users" }
126105
}
127106
},
128-
{
129-
"type": "STREAM",
130-
"stream": {
131-
"stream_state": { "updated_at": "2217-06-26T21:20:07Z" },
132-
"stream_descriptor": { "name": "project_assignments" }
133-
}
134-
},
135107
{
136108
"type": "STREAM",
137109
"stream": {
@@ -194,5 +166,81 @@
194166
"stream_state": { "to": "22170626" },
195167
"stream_descriptor": { "name": "time_team" }
196168
}
169+
},
170+
{
171+
"type": "STREAM",
172+
"stream": {
173+
"stream_state": {
174+
"states": [
175+
{
176+
"partition": { "id": 2695071, "parent_slice": {} },
177+
"cursor": { "updated_at": "2199-05-27T18:12:42Z" }
178+
}
179+
]
180+
},
181+
"stream_descriptor": { "name": "estimate_messages" }
182+
}
183+
},
184+
{
185+
"type": "STREAM",
186+
"stream": {
187+
"stream_state": {
188+
"states": [
189+
{
190+
"partition": { "id": 3758384, "parent_slice": {} },
191+
"cursor": { "updated_at": "2199-05-05T08:19:35Z" }
192+
},
193+
{
194+
"partition": { "id": 3758383, "parent_slice": {} },
195+
"cursor": { "updated_at": "2199-05-05T08:19:35Z" }
196+
},
197+
{
198+
"partition": { "id": 3758382, "parent_slice": {} },
199+
"cursor": { "updated_at": "2199-05-05T08:19:35Z" }
200+
},
201+
{
202+
"partition": { "id": 3758381, "parent_slice": {} },
203+
"cursor": { "updated_at": "2199-05-05T08:19:35Z" }
204+
},
205+
{
206+
"partition": { "id": 3758380, "parent_slice": {} },
207+
"cursor": { "updated_at": "2199-05-05T12:52:20Z" }
208+
}
209+
]
210+
},
211+
"stream_descriptor": { "name": "project_assignments" }
212+
}
213+
},
214+
{
215+
"type": "STREAM",
216+
"stream": {
217+
"stream_state": {
218+
"states": [
219+
{
220+
"partition": { "id": 28174531, "parent_slice": {} },
221+
"cursor": { "updated_at": "2199-05-25T16:46:28Z" }
222+
}
223+
]
224+
},
225+
"stream_descriptor": { "name": "invoice_messages" }
226+
}
227+
},
228+
{
229+
"type": "STREAM",
230+
"stream": {
231+
"stream_state": {
232+
"states": [
233+
{
234+
"partition": { "id": 28174545, "parent_slice": {} },
235+
"cursor": { "updated_at": "2199-05-26T09:07:06Z" }
236+
},
237+
{
238+
"partition": { "id": 28174531, "parent_slice": {} },
239+
"cursor": { "updated_at": "2199-05-26T09:06:37Z" }
240+
}
241+
]
242+
},
243+
"stream_descriptor": { "name": "invoice_payments" }
244+
}
197245
}
198246
]

‎airbyte-integrations/connectors/source-harvest/integration_tests/configured_catalog.json

+44-22
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
"json_schema": {},
77
"supported_sync_modes": ["full_refresh", "incremental"],
88
"source_defined_cursor": true,
9-
"default_cursor_field": ["updated_at"]
9+
"default_cursor_field": ["updated_at"],
10+
"source_defined_primary_key": [["id"]]
1011
},
1112
"sync_mode": "incremental",
1213
"cursor_field": ["updated_at"],
@@ -18,7 +19,8 @@
1819
"json_schema": {},
1920
"supported_sync_modes": ["full_refresh", "incremental"],
2021
"source_defined_cursor": true,
21-
"default_cursor_field": ["updated_at"]
22+
"default_cursor_field": ["updated_at"],
23+
"source_defined_primary_key": [["id"]]
2224
},
2325
"sync_mode": "incremental",
2426
"cursor_field": ["updated_at"],
@@ -39,7 +41,8 @@
3941
"json_schema": {},
4042
"supported_sync_modes": ["full_refresh", "incremental"],
4143
"source_defined_cursor": true,
42-
"default_cursor_field": ["updated_at"]
44+
"default_cursor_field": ["updated_at"],
45+
"source_defined_primary_key": [["id"]]
4346
},
4447
"sync_mode": "incremental",
4548
"cursor_field": ["updated_at"],
@@ -51,7 +54,8 @@
5154
"json_schema": {},
5255
"supported_sync_modes": ["full_refresh", "incremental"],
5356
"source_defined_cursor": true,
54-
"default_cursor_field": ["updated_at"]
57+
"default_cursor_field": ["updated_at"],
58+
"source_defined_primary_key": [["id"]]
5559
},
5660
"sync_mode": "incremental",
5761
"cursor_field": ["updated_at"],
@@ -63,7 +67,8 @@
6367
"json_schema": {},
6468
"supported_sync_modes": ["full_refresh", "incremental"],
6569
"source_defined_cursor": true,
66-
"default_cursor_field": ["updated_at"]
70+
"default_cursor_field": ["updated_at"],
71+
"source_defined_primary_key": [["id"]]
6772
},
6873
"sync_mode": "incremental",
6974
"cursor_field": ["updated_at"],
@@ -75,7 +80,8 @@
7580
"json_schema": {},
7681
"supported_sync_modes": ["full_refresh", "incremental"],
7782
"source_defined_cursor": true,
78-
"default_cursor_field": ["updated_at"]
83+
"default_cursor_field": ["updated_at"],
84+
"source_defined_primary_key": [["id"]]
7985
},
8086
"sync_mode": "incremental",
8187
"cursor_field": ["updated_at"],
@@ -87,7 +93,8 @@
8793
"json_schema": {},
8894
"supported_sync_modes": ["full_refresh", "incremental"],
8995
"source_defined_cursor": true,
90-
"default_cursor_field": ["updated_at"]
96+
"default_cursor_field": ["updated_at"],
97+
"source_defined_primary_key": [["id"]]
9198
},
9299
"sync_mode": "incremental",
93100
"cursor_field": ["updated_at"],
@@ -99,7 +106,8 @@
99106
"json_schema": {},
100107
"supported_sync_modes": ["full_refresh", "incremental"],
101108
"source_defined_cursor": true,
102-
"default_cursor_field": ["updated_at"]
109+
"default_cursor_field": ["updated_at"],
110+
"source_defined_primary_key": [["id"]]
103111
},
104112
"sync_mode": "incremental",
105113
"cursor_field": ["updated_at"],
@@ -111,7 +119,8 @@
111119
"json_schema": {},
112120
"supported_sync_modes": ["full_refresh", "incremental"],
113121
"source_defined_cursor": true,
114-
"default_cursor_field": ["updated_at"]
122+
"default_cursor_field": ["updated_at"],
123+
"source_defined_primary_key": [["id"]]
115124
},
116125
"sync_mode": "incremental",
117126
"cursor_field": ["updated_at"],
@@ -123,7 +132,8 @@
123132
"json_schema": {},
124133
"supported_sync_modes": ["full_refresh", "incremental"],
125134
"source_defined_cursor": true,
126-
"default_cursor_field": ["updated_at"]
135+
"default_cursor_field": ["updated_at"],
136+
"source_defined_primary_key": [["id"]]
127137
},
128138
"sync_mode": "incremental",
129139
"cursor_field": ["updated_at"],
@@ -135,7 +145,8 @@
135145
"json_schema": {},
136146
"supported_sync_modes": ["full_refresh", "incremental"],
137147
"source_defined_cursor": true,
138-
"default_cursor_field": ["updated_at"]
148+
"default_cursor_field": ["updated_at"],
149+
"source_defined_primary_key": [["id"]]
139150
},
140151
"sync_mode": "incremental",
141152
"cursor_field": ["updated_at"],
@@ -147,7 +158,8 @@
147158
"json_schema": {},
148159
"supported_sync_modes": ["full_refresh", "incremental"],
149160
"source_defined_cursor": true,
150-
"default_cursor_field": ["updated_at"]
161+
"default_cursor_field": ["updated_at"],
162+
"source_defined_primary_key": [["id"]]
151163
},
152164
"sync_mode": "incremental",
153165
"cursor_field": ["updated_at"],
@@ -159,7 +171,8 @@
159171
"json_schema": {},
160172
"supported_sync_modes": ["full_refresh", "incremental"],
161173
"source_defined_cursor": true,
162-
"default_cursor_field": ["updated_at"]
174+
"default_cursor_field": ["updated_at"],
175+
"source_defined_primary_key": [["id"]]
163176
},
164177
"sync_mode": "incremental",
165178
"cursor_field": ["updated_at"],
@@ -171,7 +184,8 @@
171184
"json_schema": {},
172185
"supported_sync_modes": ["full_refresh", "incremental"],
173186
"source_defined_cursor": true,
174-
"default_cursor_field": ["updated_at"]
187+
"default_cursor_field": ["updated_at"],
188+
"source_defined_primary_key": [["id"]]
175189
},
176190
"sync_mode": "incremental",
177191
"cursor_field": ["updated_at"],
@@ -183,7 +197,8 @@
183197
"json_schema": {},
184198
"supported_sync_modes": ["full_refresh", "incremental"],
185199
"source_defined_cursor": true,
186-
"default_cursor_field": ["updated_at"]
200+
"default_cursor_field": ["updated_at"],
201+
"source_defined_primary_key": [["id"]]
187202
},
188203
"sync_mode": "incremental",
189204
"cursor_field": ["updated_at"],
@@ -195,7 +210,8 @@
195210
"json_schema": {},
196211
"supported_sync_modes": ["full_refresh", "incremental"],
197212
"source_defined_cursor": true,
198-
"default_cursor_field": ["updated_at"]
213+
"default_cursor_field": ["updated_at"],
214+
"source_defined_primary_key": [["id"]]
199215
},
200216
"sync_mode": "incremental",
201217
"cursor_field": ["updated_at"],
@@ -207,7 +223,8 @@
207223
"json_schema": {},
208224
"supported_sync_modes": ["full_refresh", "incremental"],
209225
"source_defined_cursor": true,
210-
"default_cursor_field": ["updated_at"]
226+
"default_cursor_field": ["updated_at"],
227+
"source_defined_primary_key": [["id"]]
211228
},
212229
"sync_mode": "incremental",
213230
"cursor_field": ["updated_at"],
@@ -219,7 +236,8 @@
219236
"json_schema": {},
220237
"supported_sync_modes": ["full_refresh", "incremental"],
221238
"source_defined_cursor": true,
222-
"default_cursor_field": ["updated_at"]
239+
"default_cursor_field": ["updated_at"],
240+
"source_defined_primary_key": [["id"]]
223241
},
224242
"sync_mode": "incremental",
225243
"cursor_field": ["updated_at"],
@@ -249,7 +267,8 @@
249267
"json_schema": {},
250268
"supported_sync_modes": ["full_refresh", "incremental"],
251269
"source_defined_cursor": true,
252-
"default_cursor_field": ["updated_at"]
270+
"default_cursor_field": ["updated_at"],
271+
"source_defined_primary_key": [["id"]]
253272
},
254273
"sync_mode": "incremental",
255274
"cursor_field": ["updated_at"],
@@ -307,10 +326,13 @@
307326
"stream": {
308327
"name": "uninvoiced",
309328
"json_schema": {},
310-
"supported_sync_modes": ["full_refresh"]
329+
"supported_sync_modes": ["full_refresh", "incremental"],
330+
"source_defined_cursor": true,
331+
"default_cursor_field": ["to"]
311332
},
312-
"sync_mode": "full_refresh",
313-
"destination_sync_mode": "overwrite"
333+
"sync_mode": "incremental",
334+
"cursor_field": ["to"],
335+
"destination_sync_mode": "append"
314336
},
315337
{
316338
"stream": {

‎airbyte-integrations/connectors/source-harvest/integration_tests/incremental_catalog.json

+6-3
Original file line numberDiff line numberDiff line change
@@ -280,10 +280,13 @@
280280
"stream": {
281281
"name": "uninvoiced",
282282
"json_schema": {},
283-
"supported_sync_modes": ["full_refresh"]
283+
"supported_sync_modes": ["full_refresh", "incremental"],
284+
"source_defined_cursor": true,
285+
"default_cursor_field": ["to"]
284286
},
285-
"sync_mode": "full_refresh",
286-
"destination_sync_mode": "overwrite"
287+
"sync_mode": "incremental",
288+
"cursor_field": ["to"],
289+
"destination_sync_mode": "append"
287290
},
288291
{
289292
"stream": {

0 commit comments

Comments
 (0)