Skip to content

Commit d79b319

Browse files
🎉 Source Mixpanel: Beta preparation (#13372)
* Add extra mode to Source, to allow run acceptance tests * move streams into distinct modules * Add property name transformation for Export stream for avoiding collisions * Update doc * Add `date_window_size`
1 parent 815b2e1 commit d79b319

File tree

20 files changed

+1141
-880
lines changed

20 files changed

+1141
-880
lines changed

airbyte-integrations/connectors/source-mixpanel/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
1313
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
1414

1515

16-
LABEL io.airbyte.version=0.1.16
16+
LABEL io.airbyte.version=0.1.17
1717
LABEL io.airbyte.name=airbyte/source-mixpanel

airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ tests:
1616
- config_path: "secrets/config.json"
1717
configured_catalog_path: "integration_tests/configured_catalog.json"
1818
timeout_seconds: 3600
19+
empty_streams: ['export']
1920
full_refresh:
2021
- config_path: "secrets/config.json"
2122
configured_catalog_path: "integration_tests/configured_catalog.json"
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#
2+
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
from collections import defaultdict
6+
from typing import Iterable, Iterator, NamedTuple
7+
8+
9+
class TransformationResult(NamedTuple):
10+
source_name: str
11+
transformed_name: str
12+
13+
14+
def transform_property_names(property_names: Iterable[str]) -> Iterator[TransformationResult]:
15+
"""
16+
Transform property names using this rules:
17+
1. Remove leading "$" from property_name
18+
2. Resolve naming conflicts, like `userName` and `username`,
19+
that will break normalization in the future, by adding `_userName`to property name
20+
"""
21+
lowercase_collision_count = defaultdict(int)
22+
lowercase_properties = set()
23+
24+
# Sort property names for consistent result
25+
for property_name in sorted(property_names):
26+
property_name_transformed = property_name
27+
if property_name_transformed.startswith("$"):
28+
property_name_transformed = property_name_transformed[1:]
29+
30+
lowercase_property_name = property_name_transformed.lower()
31+
if lowercase_property_name in lowercase_properties:
32+
lowercase_collision_count[lowercase_property_name] += 1
33+
# Add prefix to property name
34+
prefix = "_" * lowercase_collision_count[lowercase_property_name]
35+
property_name_transformed = prefix + property_name_transformed
36+
37+
lowercase_properties.add(lowercase_property_name)
38+
yield TransformationResult(source_name=property_name, transformed_name=property_name_transformed)

0 commit comments

Comments
 (0)