Skip to content

Commit 1fcd6c4

Browse files
cjwooocjwooo
authored
Source Jira: Add Board, Epic, and Sprint-related streams and improve request caching (#6621)
* Create separate cache file per stream and add projects and start_date config * Add project id to issue record * Add projectId to issue schema * Add BoardIssues stream * Add SprintIssues stream * Add Epics stream and deduplicate state code * Add EpicIssues stream and additional fields for Issues stream * Add story points to sprint issues * Add new streams to test catalog * Update gitignore * Rename cache boolean and fix test catalog * Fix streams that depend on Issues stream * Fix sprint_issues stream * Add more fields to issues stream and format * Add option to expand issue changelogs * Remove epic_issues stream * Expand project descriptions * Show rendered fields for epics * Include project key * Include project key in issues stream * Address comments * Use CDK caching * Remove extra changes * Fix sprints stream reading from non-scrum boards * Format
1 parent c2eabc1 commit 1fcd6c4

File tree

10 files changed

+430
-123
lines changed

10 files changed

+430
-123
lines changed

‎airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json

+30
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@
3030
"sync_mode": "full_refresh",
3131
"destination_sync_mode": "overwrite"
3232
},
33+
{
34+
"stream": {
35+
"name": "board_issues",
36+
"json_schema": {},
37+
"supported_sync_modes": ["full_refresh"],
38+
"source_defined_cursor": false
39+
},
40+
"sync_mode": "full_refresh",
41+
"destination_sync_mode": "overwrite"
42+
},
3343
{
3444
"stream": {
3545
"name": "dashboards",
@@ -40,6 +50,16 @@
4050
"sync_mode": "full_refresh",
4151
"destination_sync_mode": "overwrite"
4252
},
53+
{
54+
"stream": {
55+
"name": "epics",
56+
"json_schema": {},
57+
"supported_sync_modes": ["full_refresh"],
58+
"source_defined_cursor": false
59+
},
60+
"sync_mode": "full_refresh",
61+
"destination_sync_mode": "overwrite"
62+
},
4363
{
4464
"stream": {
4565
"name": "filters",
@@ -160,6 +180,16 @@
160180
"sync_mode": "full_refresh",
161181
"destination_sync_mode": "overwrite"
162182
},
183+
{
184+
"stream": {
185+
"name": "sprint_issues",
186+
"json_schema": {},
187+
"supported_sync_modes": ["full_refresh"],
188+
"source_defined_cursor": false
189+
},
190+
"sync_mode": "full_refresh",
191+
"destination_sync_mode": "overwrite"
192+
},
163193
{
164194
"stream": {
165195
"name": "time_tracking",

‎airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
{
22
"api_token": "invalid_token",
33
"domain": "invaliddomain.atlassian.net",
4-
"email": "[email protected]"
4+
"email": "[email protected]",
5+
"projects": ["invalidproject"],
6+
"start_date": "2021-09-25T00:00:00Z"
57
}

‎airbyte-integrations/connectors/source-jira/source_jira/schemas/board_issues.json

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"type": "object",
4+
"properties": {
5+
"expand": {
6+
"type": "string"
7+
},
8+
"id": {
9+
"type": "string"
10+
},
11+
"self": {
12+
"type": "string"
13+
},
14+
"key": {
15+
"type": "string"
16+
},
17+
"fields": {
18+
"type": "object"
19+
},
20+
"boardId": {
21+
"type": "string"
22+
}
23+
}
24+
}

‎airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json

+3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
},
1717
"projectId": {
1818
"type": "string"
19+
},
20+
"projectKey": {
21+
"type": "string"
1922
}
2023
}
2124
}

‎airbyte-integrations/connectors/source-jira/source_jira/schemas/epics.json

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"type": "object",
4+
"properties": {
5+
"expand": {
6+
"type": "string",
7+
"description": "Expand options that include additional issue details in the response.",
8+
"readOnly": true,
9+
"xml": {
10+
"attribute": true
11+
}
12+
},
13+
"id": {
14+
"type": "string",
15+
"description": "The ID of the epic.",
16+
"readOnly": true
17+
},
18+
"self": {
19+
"type": "string",
20+
"description": "The URL of the epic details.",
21+
"format": "uri",
22+
"readOnly": true
23+
},
24+
"key": {
25+
"type": "string",
26+
"description": "The key of the epic.",
27+
"readOnly": true
28+
},
29+
"fields": {
30+
"type": "object",
31+
"properties": {
32+
"summary": {
33+
"type": ["string", "null"],
34+
"description": "Epic summary"
35+
},
36+
"description": {
37+
"type": ["string", "null"],
38+
"description": "Epic description"
39+
},
40+
"status": {
41+
"type": ["string", "object"],
42+
"description": "Epic status"
43+
},
44+
"updated": {
45+
"type": ["string", "null"],
46+
"format": "date-time",
47+
"description": "This field is not shown in schema / swagger, but exists in records and we use it as cursor fiekd."
48+
}
49+
},
50+
"additionalProperties": {}
51+
},
52+
"projectId": {
53+
"type": "string",
54+
"description": "The ID of the project containing the epic.",
55+
"readOnly": true
56+
},
57+
"projectKey": {
58+
"type": "string",
59+
"description": "The key of the project containing the epic.",
60+
"readOnly": true
61+
}
62+
},
63+
"additionalProperties": false
64+
}

‎airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json

+10
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,16 @@
103103
}
104104
},
105105
"additionalProperties": {}
106+
},
107+
"projectId": {
108+
"type": "string",
109+
"description": "The ID of the project containing the issue.",
110+
"readOnly": true
111+
},
112+
"projectKey": {
113+
"type": "string",
114+
"description": "The key of the project containing the issue.",
115+
"readOnly": true
106116
}
107117
},
108118
"additionalProperties": false

‎airbyte-integrations/connectors/source-jira/source_jira/schemas/sprint_issues.json

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"type": "object",
4+
"properties": {
5+
"expand": {
6+
"type": "string"
7+
},
8+
"id": {
9+
"type": "string"
10+
},
11+
"self": {
12+
"type": "string"
13+
},
14+
"key": {
15+
"type": "string"
16+
},
17+
"fields": {
18+
"type": "object"
19+
},
20+
"sprintId": {
21+
"type": "number"
22+
}
23+
}
24+
}

‎airbyte-integrations/connectors/source-jira/source_jira/source.py

+19-8
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515
from .streams import (
1616
ApplicationRoles,
1717
Avatars,
18+
BoardIssues,
1819
Boards,
1920
Dashboards,
21+
Epics,
2022
Filters,
2123
FilterSharing,
2224
Groups,
@@ -54,6 +56,7 @@
5456
ScreenSchemes,
5557
ScreenTabFields,
5658
ScreenTabs,
59+
SprintIssues,
5760
Sprints,
5861
TimeTracking,
5962
Users,
@@ -96,33 +99,40 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
9699

97100
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
98101
authenticator = self.get_authenticator(config)
99-
args = {"authenticator": authenticator, "domain": config["domain"]}
102+
args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]}
103+
incremental_args = {**args, "start_date": config["start_date"]}
100104
return [
101105
ApplicationRoles(**args),
102106
Avatars(**args),
103107
Boards(**args),
108+
BoardIssues(**incremental_args),
104109
Dashboards(**args),
110+
Epics(**incremental_args),
105111
Filters(**args),
106112
FilterSharing(**args),
107113
Groups(**args),
108-
Issues(**args),
109-
IssueComments(**args),
114+
Issues(
115+
**incremental_args,
116+
additional_fields=config.get("additional_fields", []),
117+
expand_changelog=config.get("expand_issue_changelog", False)
118+
),
119+
IssueComments(**incremental_args),
110120
IssueFields(**args),
111121
IssueFieldConfigurations(**args),
112122
IssueCustomFieldContexts(**args),
113123
IssueLinkTypes(**args),
114124
IssueNavigatorSettings(**args),
115125
IssueNotificationSchemes(**args),
116126
IssuePriorities(**args),
117-
IssueProperties(**args),
118-
IssueRemoteLinks(**args),
127+
IssueProperties(**incremental_args),
128+
IssueRemoteLinks(**incremental_args),
119129
IssueResolutions(**args),
120130
IssueSecuritySchemes(**args),
121131
IssueTypeSchemes(**args),
122132
IssueTypeScreenSchemes(**args),
123-
IssueVotes(**args),
124-
IssueWatchers(**args),
125-
IssueWorklogs(**args),
133+
IssueVotes(**incremental_args),
134+
IssueWatchers(**incremental_args),
135+
IssueWorklogs(**incremental_args),
126136
JiraSettings(**args),
127137
Labels(**args),
128138
Permissions(**args),
@@ -140,6 +150,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
140150
ScreenTabFields(**args),
141151
ScreenSchemes(**args),
142152
Sprints(**args),
153+
SprintIssues(**incremental_args),
143154
TimeTracking(**args),
144155
Users(**args),
145156
Workflows(**args),

‎airbyte-integrations/connectors/source-jira/source_jira/spec.json

+33-2
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
"$schema": "http://json-schema.org/draft-07/schema#",
55
"title": "Jira Spec",
66
"type": "object",
7-
"required": ["api_token", "domain", "email"],
8-
"additionalProperties": false,
7+
"required": ["api_token", "domain", "email", "projects", "start_date"],
8+
"additionalProperties": true,
99
"properties": {
1010
"api_token": {
1111
"type": "string",
@@ -21,6 +21,37 @@
2121
"email": {
2222
"type": "string",
2323
"description": "The user email for your Jira account"
24+
},
25+
"projects": {
26+
"type": "array",
27+
"title": "Projects",
28+
"items": {
29+
"type": "string"
30+
},
31+
"examples": ["PROJ1", "PROJ2"],
32+
"description": "Comma-separated list of Jira project keys to replicate data for"
33+
},
34+
"start_date": {
35+
"type": "string",
36+
"title": "Start Date",
37+
"description": "The date from which you'd like to replicate data for Jira in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only in the following incremental streams: issues.",
38+
"examples": ["2021-03-01T00:00:00Z"],
39+
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
40+
},
41+
"additional_fields": {
42+
"type": "array",
43+
"title": "Additional Fields",
44+
"items": {
45+
"type": "string"
46+
},
47+
"description": "Comma-separated list of additional fields to include in replicating issues",
48+
"examples": ["Field A", "Field B"]
49+
},
50+
"expand_issue_changelog": {
51+
"type": "boolean",
52+
"title": "Expand Issue Changelog",
53+
"description": "Expand the changelog when replicating issues",
54+
"default": false
2455
}
2556
}
2657
}

0 commit comments

Comments
 (0)