Skip to content

Commit 330a196

Browse files
girardasherifnadaoctavia-squidington-iii
authored
migrate sentry to config-based cdk (#15345)
* events and projects * done minus pagination * handle single records * pagination * bump min cdk version * start on unit tests * Update more unit tests * Handle extracting no records from root * additionalProperties=true * handle empty streams * skip backward compatibility tests * check on project_detail * remove unit tests * handle missing keys * delete stream classes * record extractor interface * dpath extractor * docstring * handle extract root array * Use dpath extractor * Revert "Merge branch 'alex/selectNoRecords' into alex/configbased-sentry" This reverts commit bad4dd7, reversing changes made to d9252ae. * reset to master * reset to master * reset to master * enable backward compatibility test * bump cdk version * reset * Update airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml Co-authored-by: Sherif A. Nada <[email protected]> * Use paginator * fix pagination * bump version * auto-bump connector version [ci skip] Co-authored-by: Sherif A. Nada <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 5305ad2 commit 330a196

File tree

11 files changed

+130
-341
lines changed

11 files changed

+130
-341
lines changed

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1115,7 +1115,7 @@
11151115
- sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781
11161116
name: Sentry
11171117
dockerRepository: airbyte/source-sentry
1118-
dockerImageTag: 0.1.1
1118+
dockerImageTag: 0.1.2
11191119
documentationUrl: https://docs.airbyte.io/integrations/sources/sentry
11201120
icon: sentry.svg
11211121
sourceType: api

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10766,7 +10766,7 @@
1076610766
supportsNormalization: false
1076710767
supportsDBT: false
1076810768
supported_destination_sync_modes: []
10769-
- dockerImage: "airbyte/source-sentry:0.1.1"
10769+
- dockerImage: "airbyte/source-sentry:0.1.2"
1077010770
spec:
1077110771
documentationUrl: "https://docs.airbyte.io/integrations/sources/sentry"
1077210772
connectionSpecification:
@@ -10777,7 +10777,7 @@
1077710777
- "auth_token"
1077810778
- "organization"
1077910779
- "project"
10780-
additionalProperties: false
10780+
additionalProperties: true
1078110781
properties:
1078210782
auth_token:
1078310783
type: "string"

‎airbyte-integrations/connectors/source-sentry/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry
3434
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
3535
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
3636

37-
LABEL io.airbyte.version=0.1.1
37+
LABEL io.airbyte.version=0.1.2
3838
LABEL io.airbyte.name=airbyte/source-sentry

‎airbyte-integrations/connectors/source-sentry/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from setuptools import find_packages, setup
77

88
MAIN_REQUIREMENTS = [
9-
"airbyte-cdk",
9+
"airbyte-cdk~=0.1.74",
1010
]
1111

1212
TEST_REQUIREMENTS = [

‎airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
definitions:
2+
page_size: 50
3+
schema_loader:
4+
type: JsonSchema
5+
file_path: "./source_sentry/schemas/{{ options.name }}.json"
6+
selector:
7+
type: RecordSelector
8+
extractor:
9+
type: DpathExtractor
10+
field_pointer: []
11+
requester:
12+
type: HttpRequester
13+
name: "{{ options['name'] }}"
14+
url_base: "https://{{ config.hostname }}/api/0/"
15+
http_method: "GET"
16+
authenticator:
17+
type: "BearerAuthenticator"
18+
api_token: "{{ config.auth_token }}"
19+
paginator:
20+
type: LimitPaginator
21+
url_base: "*ref(definitions.requester.url_base)"
22+
page_size: "*ref(definitions.page_size)"
23+
limit_option:
24+
inject_into: "request_parameter"
25+
field_name: ""
26+
page_token_option:
27+
inject_into: "request_parameter"
28+
field_name: "cursor"
29+
pagination_strategy:
30+
type: "CursorPagination"
31+
cursor_value: "{{ headers.link.next.cursor }}"
32+
stop_condition: "{{ headers.link.next.results != 'true' }}"
33+
retriever:
34+
type: SimpleRetriever
35+
name: "{{ options['name'] }}"
36+
primary_key: "{{ options['primary_key'] }}"
37+
38+
streams:
39+
- type: DeclarativeStream
40+
$options:
41+
# https://docs.sentry.io/api/events/list-a-projects-events/
42+
name: "events"
43+
primary_key: "id"
44+
schema_loader:
45+
$ref: "*ref(definitions.schema_loader)"
46+
retriever:
47+
$ref: "*ref(definitions.retriever)"
48+
record_selector:
49+
$ref: "*ref(definitions.selector)"
50+
requester:
51+
$ref: "*ref(definitions.requester)"
52+
path: "projects/{{config.organization}}/{{config.project}}/events/"
53+
request_options_provider:
54+
request_parameters:
55+
full: "true"
56+
paginator:
57+
$ref: "*ref(definitions.paginator)"
58+
- type: DeclarativeStream
59+
$options:
60+
name: "issues"
61+
primary_key: "id"
62+
schema_loader:
63+
$ref: "*ref(definitions.schema_loader)"
64+
retriever:
65+
$ref: "*ref(definitions.retriever)"
66+
record_selector:
67+
$ref: "*ref(definitions.selector)"
68+
requester:
69+
$ref: "*ref(definitions.requester)"
70+
path: "projects/{{config.organization}}/{{config.project}}/issues/"
71+
request_options_provider:
72+
request_parameters:
73+
statsPeriod: ""
74+
query: ""
75+
paginator:
76+
$ref: "*ref(definitions.paginator)"
77+
- type: DeclarativeStream
78+
$options:
79+
name: "projects"
80+
primary_key: "id"
81+
schema_loader:
82+
$ref: "*ref(definitions.schema_loader)"
83+
retriever:
84+
$ref: "*ref(definitions.retriever)"
85+
record_selector:
86+
$ref: "*ref(definitions.selector)"
87+
requester:
88+
$ref: "*ref(definitions.requester)"
89+
path: "projects/"
90+
paginator:
91+
$ref: "*ref(definitions.paginator)"
92+
- type: DeclarativeStream
93+
$options:
94+
name: "project_detail"
95+
primary_key: "id"
96+
schema_loader:
97+
$ref: "*ref(definitions.schema_loader)"
98+
retriever:
99+
$ref: "*ref(definitions.retriever)"
100+
record_selector:
101+
$ref: "*ref(definitions.selector)"
102+
requester:
103+
$ref: "*ref(definitions.requester)"
104+
path: "projects/{{config.organization}}/{{config.project}}/"
105+
paginator:
106+
type: NoPagination
107+
check:
108+
type: CheckStream
109+
stream_names: ["project_detail"]

‎airbyte-integrations/connectors/source-sentry/source_sentry/source.py

Lines changed: 10 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,43 +2,16 @@
22
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
33
#
44

5+
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
56

6-
from typing import Any, List, Mapping, Tuple
7+
"""
8+
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
9+
source connector.
10+
WARNING: Do not modify this file.
11+
"""
712

8-
from airbyte_cdk.models import SyncMode
9-
from airbyte_cdk.sources import AbstractSource
10-
from airbyte_cdk.sources.streams import Stream
11-
from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
1213

13-
from .streams import Events, Issues, ProjectDetail, Projects
14-
15-
16-
# Source
17-
class SourceSentry(AbstractSource):
18-
def check_connection(self, logger, config) -> Tuple[bool, Any]:
19-
try:
20-
projects_stream = Projects(
21-
authenticator=TokenAuthenticator(token=config["auth_token"]),
22-
hostname=config.get("hostname"),
23-
)
24-
next(projects_stream.read_records(sync_mode=SyncMode.full_refresh))
25-
return True, None
26-
except Exception as e:
27-
return False, e
28-
29-
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
30-
stream_args = {
31-
"authenticator": TokenAuthenticator(token=config["auth_token"]),
32-
"hostname": config.get("hostname"),
33-
}
34-
project_stream_args = {
35-
**stream_args,
36-
"organization": config["organization"],
37-
"project": config["project"],
38-
}
39-
return [
40-
Events(**project_stream_args),
41-
Issues(**project_stream_args),
42-
ProjectDetail(**project_stream_args),
43-
Projects(**stream_args),
44-
]
14+
# Declarative Source
15+
class SourceSentry(YamlDeclarativeSource):
16+
def __init__(self):
17+
super().__init__(**{"path_to_yaml": "./source_sentry/sentry.yaml"})

‎airbyte-integrations/connectors/source-sentry/source_sentry/spec.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"title": "Sentry Spec",
66
"type": "object",
77
"required": ["auth_token", "organization", "project"],
8-
"additionalProperties": false,
8+
"additionalProperties": true,
99
"properties": {
1010
"auth_token": {
1111
"type": "string",

‎airbyte-integrations/connectors/source-sentry/source_sentry/streams.py

Lines changed: 0 additions & 158 deletions
This file was deleted.

0 commit comments

Comments
 (0)