Skip to content

Commit e40ee3f

Browse files
roman-yermilov-glgrubberroctavia-squidington-iii
authored
🎉Source Google Analytics Data Api: code refactoring (#15229)
* Source: google analytics data api refactoring * google-analytics-data-api.md Signed-off-by: Sergey Chvalyuk <[email protected]> * updated spec: added custom reports field * doclint Signed-off-by: Sergey Chvalyuk <[email protected]> * auto-bump connector version [ci skip] Signed-off-by: Sergey Chvalyuk <[email protected]> Co-authored-by: Sergey Chvalyuk <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
1 parent f4adef7 commit e40ee3f

File tree

22 files changed

+1202
-305
lines changed

22 files changed

+1202
-305
lines changed

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@
363363
- name: Google Analytics Data API
364364
sourceDefinitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a
365365
dockerRepository: airbyte/source-google-analytics-data-api
366-
dockerImageTag: 0.0.2
366+
dockerImageTag: 0.0.3
367367
documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4
368368
icon: google-analytics.svg
369369
sourceType: api

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

Lines changed: 102 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -3191,7 +3191,7 @@
31913191
oauthFlowOutputParameters:
31923192
- - "access_token"
31933193
- - "refresh_token"
3194-
- dockerImage: "airbyte/source-google-analytics-data-api:0.0.2"
3194+
- dockerImage: "airbyte/source-google-analytics-data-api:0.0.3"
31953195
spec:
31963196
documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-v4"
31973197
connectionSpecification:
@@ -3200,55 +3200,124 @@
32003200
type: "object"
32013201
required:
32023202
- "property_id"
3203-
- "json_credentials"
3204-
- "report_name"
3205-
- "dimensions"
3206-
- "metrics"
32073203
- "date_ranges_start_date"
3208-
- "date_ranges_end_date"
3204+
additionalProperties: true
32093205
properties:
32103206
property_id:
32113207
type: "string"
32123208
title: "Property ID"
32133209
description: "A Google Analytics GA4 property identifier whose events are\
32143210
\ tracked. Specified in the URL path and not the body"
32153211
order: 1
3216-
json_credentials:
3217-
type: "string"
3218-
title: "JSON Credentials"
3219-
description: "The JSON key of the Service Account to use for authorization"
3220-
airbyte_secret: true
3221-
order: 2
3222-
report_name:
3223-
type: "string"
3224-
title: "Report Name"
3225-
description: "The report name"
3226-
order: 3
3227-
dimensions:
3228-
type: "string"
3229-
title: "Dimensions"
3230-
description: "Comma seprated report dimensions https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#dimensions"
3231-
order: 4
3232-
metrics:
3233-
type: "string"
3234-
title: "Metrics"
3235-
description: "Comma seprated report metrics https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#metrics"
3236-
order: 5
3212+
credentials:
3213+
order: 0
3214+
type: "object"
3215+
title: "Credentials"
3216+
description: "Credentials for the service"
3217+
oneOf:
3218+
- title: "Authenticate via Google (Oauth)"
3219+
type: "object"
3220+
required:
3221+
- "client_id"
3222+
- "client_secret"
3223+
- "refresh_token"
3224+
properties:
3225+
auth_type:
3226+
type: "string"
3227+
const: "Client"
3228+
order: 0
3229+
client_id:
3230+
title: "Client ID"
3231+
type: "string"
3232+
description: "The Client ID of your Google Analytics developer application."
3233+
airbyte_secret: true
3234+
order: 1
3235+
client_secret:
3236+
title: "Client Secret"
3237+
type: "string"
3238+
description: "The Client Secret of your Google Analytics developer\
3239+
\ application."
3240+
airbyte_secret: true
3241+
order: 2
3242+
refresh_token:
3243+
title: "Refresh Token"
3244+
type: "string"
3245+
description: "The token for obtaining a new access token."
3246+
airbyte_secret: true
3247+
order: 3
3248+
access_token:
3249+
title: "Access Token (Optional)"
3250+
type: "string"
3251+
description: "Access Token for making authenticated requests."
3252+
airbyte_secret: true
3253+
order: 4
3254+
- type: "object"
3255+
title: "Service Account Key Authentication"
3256+
required:
3257+
- "credentials_json"
3258+
properties:
3259+
auth_type:
3260+
type: "string"
3261+
const: "Service"
3262+
order: 0
3263+
credentials_json:
3264+
title: "Service Account JSON Key"
3265+
type: "string"
3266+
description: "The JSON key of the service account to use for authorization"
3267+
examples:
3268+
- "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\
3269+
\ \"private_key_id\": YOUR_PRIVATE_KEY, ... }"
3270+
airbyte_secret: true
32373271
date_ranges_start_date:
32383272
type: "string"
32393273
title: "Date Range Start Date"
32403274
description: "The start date. One of the values Ndaysago, yesterday, today\
32413275
\ or in the format YYYY-MM-DD"
3242-
order: 6
3243-
date_ranges_end_date:
3276+
order: 2
3277+
custom_reports:
3278+
order: 3
32443279
type: "string"
3245-
title: "Date Range End Date"
3246-
description: "The end date. One of the values Ndaysago, yesterday, today\
3247-
\ or in the format YYYY-MM-DD"
3248-
order: 7
3280+
title: "Custom Reports (Optional)"
3281+
description: "A JSON array describing the custom reports you want to sync\
3282+
\ from Google Analytics. See <a href=\"https://docs.airbyte.io/integrations/sources/google-analytics-v4#data-processing-latency\"\
3283+
>the docs</a> for more information about the exact format you can use\
3284+
\ to fill out this field."
3285+
window_in_days:
3286+
type: "integer"
3287+
title: "Data request time increment in days (Optional)"
3288+
description: "The time increment used by the connector when requesting data\
3289+
\ from the Google Analytics API. More information is available in the\
3290+
\ <a href=\"https://docs.airbyte.com/integrations/sources/google-analytics-v4/#sampling-in-reports\"\
3291+
>the docs</a>. The bigger this value is, the faster the sync will be,\
3292+
\ but the more likely that sampling will be applied to your data, potentially\
3293+
\ causing inaccuracies in the returned results. We recommend setting this\
3294+
\ to 1 unless you have a hard requirement to make the sync faster at the\
3295+
\ expense of accuracy. The minimum allowed value for this field is 1,\
3296+
\ and the maximum is 364. "
3297+
examples:
3298+
- 30
3299+
- 60
3300+
- 90
3301+
- 120
3302+
- 200
3303+
- 364
3304+
default: 1
3305+
order: 4
32493306
supportsNormalization: false
32503307
supportsDBT: false
32513308
supported_destination_sync_modes: []
3309+
authSpecification:
3310+
auth_type: "oauth2.0"
3311+
oauth2Specification:
3312+
rootObject:
3313+
- "credentials"
3314+
- "0"
3315+
oauthFlowInitParameters:
3316+
- - "client_id"
3317+
- - "client_secret"
3318+
oauthFlowOutputParameters:
3319+
- - "access_token"
3320+
- - "refresh_token"
32523321
- dockerImage: "airbyte/source-google-directory:0.1.9"
32533322
spec:
32543323
documentationUrl: "https://docs.airbyte.io/integrations/sources/google-directory"

‎airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,5 @@ COPY source_google_analytics_data_api ./source_google_analytics_data_api
2828
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
2929
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
3030

31-
LABEL io.airbyte.version=0.0.2
31+
LABEL io.airbyte.version=0.0.3
3232
LABEL io.airbyte.name=airbyte/source-google-analytics-data-api

‎airbyte-integrations/connectors/source-google-analytics-data-api/Makefile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
docker_image := airbyte/$(notdir $(CURDIR)):dev
2+
3+
run-build:
4+
docker build . -t ${docker_image}
5+
6+
spec:
7+
@docker run --rm $(docker_image) spec | jq
8+
9+
check:
10+
@docker run --rm -v $(PWD)/secrets:/secrets $(docker_image) check --config /secrets/config.json | jq
11+
12+
discover:
13+
@docker run --rm -v $(PWD)/secrets:/secrets $(docker_image) discover --config /secrets/config.json | jq
14+
15+
read:
16+
@docker run --rm -v $(PWD)/secrets:/secrets -v $(PWD)/integration_tests:/integration_tests $(docker_image) read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json | jq
17+
18+
unittest-local:
19+
@python -m pytest unit_tests

‎airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,21 @@ tests:
1111
status: "failed"
1212
discovery:
1313
- config_path: "secrets/config.json"
14+
backward_compatibility_tests_config:
15+
disable_for_version: "0.0.2"
1416
basic_read:
1517
- config_path: "secrets/config.json"
1618
configured_catalog_path: "integration_tests/configured_catalog.json"
1719
empty_streams: []
1820
full_refresh:
1921
- config_path: "secrets/config.json"
2022
configured_catalog_path: "integration_tests/configured_catalog.json"
23+
ignored_fields:
24+
"daily_active_users": ["uuid"]
25+
"weekly_active_users": ["uuid"]
26+
"four_weekly_active_users": ["uuid"]
27+
"devices": ["uuid"]
28+
"locations": ["uuid"]
29+
"pages": ["uuid"]
30+
"traffic_sources": ["uuid"]
31+
"website_overview": ["uuid"]

‎airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,90 @@
22
"streams": [
33
{
44
"stream": {
5-
"name": "Analytics Report",
5+
"name": "daily_active_users",
66
"json_schema": {},
7-
"supported_sync_modes": ["full_refresh"],
7+
"supported_sync_modes": ["incremental"],
88
"source_defined_cursor": false,
9-
"default_cursor_field": ["column_name"]
9+
"default_cursor_field": ["date"]
1010
},
11-
"sync_mode": "full_refresh",
11+
"sync_mode": "incremental",
12+
"destination_sync_mode": "overwrite"
13+
},
14+
{
15+
"stream": {
16+
"name": "weekly_active_users",
17+
"json_schema": {},
18+
"supported_sync_modes": ["incremental"],
19+
"source_defined_cursor": false,
20+
"default_cursor_field": ["date"]
21+
},
22+
"sync_mode": "incremental",
23+
"destination_sync_mode": "overwrite"
24+
},
25+
{
26+
"stream": {
27+
"name": "four_weekly_active_users",
28+
"json_schema": {},
29+
"supported_sync_modes": ["incremental"],
30+
"source_defined_cursor": false,
31+
"default_cursor_field": ["date"]
32+
},
33+
"sync_mode": "incremental",
34+
"destination_sync_mode": "overwrite"
35+
},
36+
{
37+
"stream": {
38+
"name": "devices",
39+
"json_schema": {},
40+
"supported_sync_modes": ["incremental"],
41+
"source_defined_cursor": false,
42+
"default_cursor_field": ["date"]
43+
},
44+
"sync_mode": "incremental",
45+
"destination_sync_mode": "overwrite"
46+
},
47+
{
48+
"stream": {
49+
"name": "locations",
50+
"json_schema": {},
51+
"supported_sync_modes": ["incremental"],
52+
"source_defined_cursor": false,
53+
"default_cursor_field": ["date"]
54+
},
55+
"sync_mode": "incremental",
56+
"destination_sync_mode": "overwrite"
57+
},
58+
{
59+
"stream": {
60+
"name": "pages",
61+
"json_schema": {},
62+
"supported_sync_modes": ["incremental"],
63+
"source_defined_cursor": false,
64+
"default_cursor_field": ["date"]
65+
},
66+
"sync_mode": "incremental",
67+
"destination_sync_mode": "overwrite"
68+
},
69+
{
70+
"stream": {
71+
"name": "traffic_sources",
72+
"json_schema": {},
73+
"supported_sync_modes": ["incremental"],
74+
"source_defined_cursor": false,
75+
"default_cursor_field": ["date"]
76+
},
77+
"sync_mode": "incremental",
78+
"destination_sync_mode": "overwrite"
79+
},
80+
{
81+
"stream": {
82+
"name": "website_overview",
83+
"json_schema": {},
84+
"supported_sync_modes": ["incremental"],
85+
"source_defined_cursor": false,
86+
"default_cursor_field": ["date"]
87+
},
88+
"sync_mode": "incremental",
1289
"destination_sync_mode": "overwrite"
1390
}
1491
]

‎airbyte-integrations/connectors/source-google-analytics-data-api/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
from setuptools import find_packages, setup
77

8-
MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-analytics-data==0.11.2"]
8+
MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-analytics-data==0.11.2", "PyJWT==2.4.0", "cryptography==37.0.4", "requests==2.28.1"]
99

1010
TEST_REQUIREMENTS = [
1111
"pytest~=6.1",

0 commit comments

Comments
 (0)