Skip to content

Commit 4fa05b7

Browse files
avidaavida
authored
🎉 Pipedrive source: add organizations stream (#5943)
* Pipedrive source: add organizatins stream
1 parent b53d826 commit 4fa05b7

File tree

11 files changed

+239
-10
lines changed

11 files changed

+239
-10
lines changed

‎airbyte-integrations/connectors/source-pipedrive/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ RUN pip install .
1212
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
1313
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
1414

15-
LABEL io.airbyte.version=0.1.3
15+
LABEL io.airbyte.version=0.1.4
1616
LABEL io.airbyte.name=airbyte/source-pipedrive

‎airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ tests:
1212
basic_read:
1313
- config_path: "secrets/config.json"
1414
configured_catalog_path: "integration_tests/configured_catalog.json"
15+
expect_records:
16+
path: "integration_tests/expected_records.txt"
1517
incremental:
1618
- config_path: "secrets/config.json"
1719
configured_catalog_path: "integration_tests/configured_catalog.json"

‎airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json

+3
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,8 @@
1616
},
1717
"users": {
1818
"modified": "2217-06-26T21:20:07Z"
19+
},
20+
"organizations": {
21+
"update_time": "2217-06-26T21:20:07Z"
1922
}
2023
}

‎airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json

+12
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,18 @@
4242
"cursor_field": ["update_time"],
4343
"destination_sync_mode": "append"
4444
},
45+
{
46+
"stream": {
47+
"name": "organizations",
48+
"json_schema": {},
49+
"supported_sync_modes": ["full_refresh", "incremental"],
50+
"source_defined_cursor": true,
51+
"default_cursor_field": ["update_time"]
52+
},
53+
"sync_mode": "incremental",
54+
"cursor_field": ["update_time"],
55+
"destination_sync_mode": "append"
56+
},
4557
{
4658
"stream": {
4759
"name": "persons",

‎airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"stream": "organizations", "data": {"id": 1, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization1", "open_deals_count": 10, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 2, "activities_count": 2, "done_activities_count": 0, "undone_activities_count": 2, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2021-07-06 15:05:14", "add_time": "2021-07-06 13:20:52", "visible_to": "3", "next_activity_date": "2021-07-06", "next_activity_time": null, "next_activity_id": 1, "last_activity_id": null, "last_activity_date": null, "label": null, "address": null, "address_subpremise": null, "address_street_number": null, "address_route": null, "address_sublocality": null, "address_locality": null, "address_admin_area_level_1": null, "address_admin_area_level_2": null, "address_country": null, "address_postal_code": null, "address_formatted_address": null, "cc_email": "[email protected]", "owner_name": "Team Airbyte"}, "emitted_at": 1631182239000}

‎airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
},
5656
"next_activity_date": {
5757
"type": ["null", "string"],
58-
"format": "date-time"
58+
"format": "date"
5959
},
6060
"next_activity_time": {
6161
"type": ["null", "string"]
@@ -124,7 +124,7 @@
124124
},
125125
"expected_close_date": {
126126
"type": ["null", "string"],
127-
"format": "date-time"
127+
"format": "date"
128128
},
129129
"last_incoming_mail_time": {
130130
"type": ["null", "string"],

‎airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json

+201
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"type": ["null", "object"],
4+
"properties": {
5+
"id": {
6+
"type": ["null", "number"]
7+
},
8+
"company_id": {
9+
"type": ["null", "number"]
10+
},
11+
"owner_id": {
12+
"type": ["null", "object", "number"],
13+
"properties": {
14+
"id": {
15+
"type": ["null", "number"]
16+
},
17+
"name": {
18+
"type": ["null", "string"]
19+
},
20+
"email": {
21+
"type": ["null", "string"]
22+
},
23+
"has_pic": {
24+
"type": ["null", "number"]
25+
},
26+
"pic_hash": {
27+
"type": ["null", "string"]
28+
},
29+
"active_flag": {
30+
"type": ["null", "boolean"]
31+
},
32+
"value": {
33+
"type": ["null", "number"]
34+
}
35+
}
36+
},
37+
"name": {
38+
"type": ["null", "string"]
39+
},
40+
"open_deals_count": {
41+
"type": ["null", "number"]
42+
},
43+
"related_open_deals_count": {
44+
"type": ["null", "number"]
45+
},
46+
"closed_deals_count": {
47+
"type": ["null", "number"]
48+
},
49+
"related_closed_deals_count": {
50+
"type": ["null", "number"]
51+
},
52+
"email_messages_count": {
53+
"type": ["null", "number"]
54+
},
55+
"people_count": {
56+
"type": ["null", "number"]
57+
},
58+
"activities_count": {
59+
"type": ["null", "number"]
60+
},
61+
"done_activities_count": {
62+
"type": ["null", "number"]
63+
},
64+
"undone_activities_count": {
65+
"type": ["null", "number"]
66+
},
67+
"files_count": {
68+
"type": ["null", "number"]
69+
},
70+
"notes_count": {
71+
"type": ["null", "number"]
72+
},
73+
"followers_count": {
74+
"type": ["null", "number"]
75+
},
76+
"won_deals_count": {
77+
"type": ["null", "number"]
78+
},
79+
"related_won_deals_count": {
80+
"type": ["null", "number"]
81+
},
82+
"lost_deals_count": {
83+
"type": ["null", "number"]
84+
},
85+
"related_lost_deals_count": {
86+
"type": ["null", "number"]
87+
},
88+
"active_flag": {
89+
"type": ["null", "boolean"]
90+
},
91+
"picture_id": {
92+
"type": ["null", "object"],
93+
"properties": {
94+
"item_type": {
95+
"type": ["null", "string"]
96+
},
97+
"item_id": {
98+
"type": ["null", "number"]
99+
},
100+
"active_flag": {
101+
"type": ["null", "boolean"]
102+
},
103+
"add_time": {
104+
"type": ["null", "string"]
105+
},
106+
"update_time": {
107+
"type": ["null", "string"]
108+
},
109+
"added_by_user_id": {
110+
"type": ["null", "number"]
111+
},
112+
"pictures": {
113+
"type": ["null", "object"],
114+
"properties": {
115+
"128": {
116+
"type": ["null", "string"]
117+
},
118+
"512": {
119+
"type": ["null", "string"]
120+
}
121+
}
122+
},
123+
"value": {
124+
"type": ["null", "number"]
125+
}
126+
}
127+
},
128+
"country_code": {
129+
"type": ["null", "string"]
130+
},
131+
"first_char": {
132+
"type": ["null", "string"]
133+
},
134+
"update_time": {
135+
"type": ["null", "string"]
136+
},
137+
"add_time": {
138+
"type": ["null", "string"]
139+
},
140+
"visible_to": {
141+
"type": ["null", "number", "string"]
142+
},
143+
"next_activity_date": {
144+
"type": ["null", "string"]
145+
},
146+
"next_activity_time": {
147+
"type": ["null", "string"]
148+
},
149+
"next_activity_id": {
150+
"type": ["null", "number"]
151+
},
152+
"last_activity_id": {
153+
"type": ["null", "number"]
154+
},
155+
"last_activity_date": {
156+
"type": ["null", "string"]
157+
},
158+
"label": {
159+
"type": ["null", "number"]
160+
},
161+
"address": {
162+
"type": ["null", "string"]
163+
},
164+
"address_subpremise": {
165+
"type": ["null", "string"]
166+
},
167+
"address_street_number": {
168+
"type": ["null", "string"]
169+
},
170+
"address_route": {
171+
"type": ["null", "string"]
172+
},
173+
"address_sublocality": {
174+
"type": ["null", "string"]
175+
},
176+
"address_locality": {
177+
"type": ["null", "string"]
178+
},
179+
"address_admin_area_level_1": {
180+
"type": ["null", "string"]
181+
},
182+
"address_admin_area_level_2": {
183+
"type": ["null", "string"]
184+
},
185+
"address_country": {
186+
"type": ["null", "string"]
187+
},
188+
"address_postal_code": {
189+
"type": ["null", "string"]
190+
},
191+
"address_formatted_address": {
192+
"type": ["null", "string"]
193+
},
194+
"owner_name": {
195+
"type": ["null", "string"]
196+
},
197+
"cc_email": {
198+
"type": ["null", "string"]
199+
}
200+
}
201+
}

‎airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@
137137
},
138138
"next_activity_date": {
139139
"type": ["null", "string"],
140-
"format": "date-time"
140+
"format": "date"
141141
},
142142
"next_activity_time": {
143143
"type": ["null", "string"],

‎airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
from airbyte_cdk.models import SyncMode
3131
from airbyte_cdk.sources import AbstractSource
3232
from airbyte_cdk.sources.streams import Stream
33-
from source_pipedrive.streams import Activities, ActivityFields, Deals, Leads, Persons, Pipelines, Stages, Users
33+
from source_pipedrive.streams import Activities, ActivityFields, Deals, Leads, Organizations, Persons, Pipelines, Stages, Users
3434

3535

3636
class SourcePipedrive(AbstractSource):
@@ -54,6 +54,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
5454
ActivityFields(**stream_kwargs),
5555
Deals(**incremental_stream_kwargs),
5656
Leads(**stream_kwargs),
57+
Organizations(**incremental_stream_kwargs),
5758
Persons(**incremental_stream_kwargs),
5859
Pipelines(**incremental_stream_kwargs),
5960
Stages(**incremental_stream_kwargs),

‎airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py

+12-4
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,11 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
9898
"""
9999
records = response.json().get(self.data_field) or []
100100
for record in records:
101-
if record.get(self.data_field):
102-
yield record.get(self.data_field)
103-
else:
104-
yield record
101+
record = record.get(self.data_field) or record
102+
if self.primary_key in record and record[self.primary_key] is None:
103+
# Convert "id: null" fields to "id: 0" since id is primary key and SAT checks if it is not null.
104+
record[self.primary_key] = 0
105+
yield record
105106

106107
def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
107108
"""
@@ -138,6 +139,13 @@ class ActivityFields(PipedriveStream):
138139
"""https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields"""
139140

140141

142+
class Organizations(PipedriveStream):
143+
"""
144+
API docs: https://developers.pipedrive.com/docs/api/v1/Organizations#getOrganizations,
145+
retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents
146+
"""
147+
148+
141149
class Persons(PipedriveStream):
142150
"""
143151
API docs: https://developers.pipedrive.com/docs/api/v1/Persons#getPersons,

‎docs/integrations/sources/pipedrive.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,8 @@ See [How to find the API token](https://pipedrive.readme.io/docs/how-to-find-the
7676

7777
| Version | Date | Pull Request | Subject |
7878
| :------ | :-------- | :----- | :------ |
79+
| 0.1.4 | 2021-08-26 | [5943](https://github.com/airbytehq/airbyte/pull/5943) | Add organizations stream |
7980
| 0.1.3 | 2021-08-26 | [5642](https://github.com/airbytehq/airbyte/pull/5642) | Remove date-time from deals stream |
8081
| 0.1.2 | 2021-07-23 | [4912](https://github.com/airbytehq/airbyte/pull/4912) | Update money type to support floating point |
8182
| 0.1.1 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Update spec.json |
82-
| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Release Pipedrive connector! |
83+
| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Release Pipedrive connector! |

0 commit comments

Comments
 (0)