Skip to content

Commit d452403

Browse files
Daemonxiaozhangyangyumarcosmarxm
authored
🎉 New Destination: TiDB (#15592)
* Add new destination-tidb * support sync * Add normalization-tidb * fix failed tests * Add unnest marco * fmt * Add new destination-tidb * support sync * Add normalization-tidb * fix failed tests * Add unnest marco * fmt * fmt * fix integration test * Update docs/integrations/destinations/tidb.md Co-authored-by: Xiang Zhang <[email protected]> * Update doc * Update doc * Update doc * bump normalization version * update normalization changelog * run format * add dest def * generat spec Co-authored-by: Xiang Zhang <[email protected]> Co-authored-by: Marcos Marx <[email protected]> Co-authored-by: marcosmarxm <[email protected]>
1 parent 3cfa489 commit d452403

File tree

50 files changed

+2334
-12
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2334
-12
lines changed

‎airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,3 +315,9 @@
315315
documentationUrl: https://docs.airbyte.io/integrations/destinations/local-sqlite
316316
icon: sqlite.svg
317317
releaseStage: alpha
318+
- name: TiDB
319+
destinationDefinitionId: 06ec60c7-7468-45c0-91ac-174f6e1a788b
320+
dockerRepository: airbyte/destination-tidb
321+
dockerImageTag: 0.1.0
322+
documentationUrl: https://docs.airbyte.io/integrations/destinations/tidb
323+
releaseStage: alpha

‎airbyte-config/init/src/main/resources/seed/destination_specs.yaml

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5325,3 +5325,169 @@
53255325
supported_destination_sync_modes:
53265326
- "overwrite"
53275327
- "append"
5328+
- dockerImage: "airbyte/destination-tidb:0.1.0"
5329+
spec:
5330+
documentationUrl: "https://docs.airbyte.io/integrations/destinations/tidb"
5331+
connectionSpecification:
5332+
$schema: "http://json-schema.org/draft-07/schema#"
5333+
title: "TiDB Destination Spec"
5334+
type: "object"
5335+
required:
5336+
- "host"
5337+
- "port"
5338+
- "username"
5339+
- "database"
5340+
additionalProperties: true
5341+
properties:
5342+
host:
5343+
title: "Host"
5344+
description: "Hostname of the database."
5345+
type: "string"
5346+
order: 0
5347+
port:
5348+
title: "Port"
5349+
description: "Port of the database."
5350+
type: "integer"
5351+
minimum: 0
5352+
maximum: 65536
5353+
default: 4000
5354+
examples:
5355+
- "4000"
5356+
order: 1
5357+
database:
5358+
title: "Database"
5359+
description: "Name of the database."
5360+
type: "string"
5361+
order: 2
5362+
username:
5363+
title: "User"
5364+
description: "Username to use to access the database."
5365+
type: "string"
5366+
order: 3
5367+
password:
5368+
title: "Password"
5369+
description: "Password associated with the username."
5370+
type: "string"
5371+
airbyte_secret: true
5372+
default: ""
5373+
order: 4
5374+
ssl:
5375+
title: "SSL Connection"
5376+
description: "Encrypt data using SSL."
5377+
type: "boolean"
5378+
default: false
5379+
order: 5
5380+
jdbc_url_params:
5381+
description: "Additional properties to pass to the JDBC URL string when\
5382+
\ connecting to the database formatted as 'key=value' pairs separated\
5383+
\ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)."
5384+
title: "JDBC URL Params"
5385+
type: "string"
5386+
order: 6
5387+
tunnel_method:
5388+
type: "object"
5389+
title: "SSH Tunnel Method"
5390+
description: "Whether to initiate an SSH tunnel before connecting to the\
5391+
\ database, and if so, which kind of authentication to use."
5392+
oneOf:
5393+
- title: "No Tunnel"
5394+
required:
5395+
- "tunnel_method"
5396+
properties:
5397+
tunnel_method:
5398+
description: "No ssh tunnel needed to connect to database"
5399+
type: "string"
5400+
const: "NO_TUNNEL"
5401+
order: 0
5402+
- title: "SSH Key Authentication"
5403+
required:
5404+
- "tunnel_method"
5405+
- "tunnel_host"
5406+
- "tunnel_port"
5407+
- "tunnel_user"
5408+
- "ssh_key"
5409+
properties:
5410+
tunnel_method:
5411+
description: "Connect through a jump server tunnel host using username\
5412+
\ and ssh key"
5413+
type: "string"
5414+
const: "SSH_KEY_AUTH"
5415+
order: 0
5416+
tunnel_host:
5417+
title: "SSH Tunnel Jump Server Host"
5418+
description: "Hostname of the jump server host that allows inbound\
5419+
\ ssh tunnel."
5420+
type: "string"
5421+
order: 1
5422+
tunnel_port:
5423+
title: "SSH Connection Port"
5424+
description: "Port on the proxy/jump server that accepts inbound ssh\
5425+
\ connections."
5426+
type: "integer"
5427+
minimum: 0
5428+
maximum: 65536
5429+
default: 22
5430+
examples:
5431+
- "22"
5432+
order: 2
5433+
tunnel_user:
5434+
title: "SSH Login Username"
5435+
description: "OS-level username for logging into the jump server host."
5436+
type: "string"
5437+
order: 3
5438+
ssh_key:
5439+
title: "SSH Private Key"
5440+
description: "OS-level user account ssh key credentials in RSA PEM\
5441+
\ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )"
5442+
type: "string"
5443+
airbyte_secret: true
5444+
multiline: true
5445+
order: 4
5446+
- title: "Password Authentication"
5447+
required:
5448+
- "tunnel_method"
5449+
- "tunnel_host"
5450+
- "tunnel_port"
5451+
- "tunnel_user"
5452+
- "tunnel_user_password"
5453+
properties:
5454+
tunnel_method:
5455+
description: "Connect through a jump server tunnel host using username\
5456+
\ and password authentication"
5457+
type: "string"
5458+
const: "SSH_PASSWORD_AUTH"
5459+
order: 0
5460+
tunnel_host:
5461+
title: "SSH Tunnel Jump Server Host"
5462+
description: "Hostname of the jump server host that allows inbound\
5463+
\ ssh tunnel."
5464+
type: "string"
5465+
order: 1
5466+
tunnel_port:
5467+
title: "SSH Connection Port"
5468+
description: "Port on the proxy/jump server that accepts inbound ssh\
5469+
\ connections."
5470+
type: "integer"
5471+
minimum: 0
5472+
maximum: 65536
5473+
default: 22
5474+
examples:
5475+
- "22"
5476+
order: 2
5477+
tunnel_user:
5478+
title: "SSH Login Username"
5479+
description: "OS-level username for logging into the jump server host"
5480+
type: "string"
5481+
order: 3
5482+
tunnel_user_password:
5483+
title: "Password"
5484+
description: "OS-level password for logging into the jump server host"
5485+
type: "string"
5486+
airbyte_secret: true
5487+
order: 4
5488+
supportsIncremental: true
5489+
supportsNormalization: true
5490+
supportsDBT: true
5491+
supported_destination_sync_modes:
5492+
- "overwrite"
5493+
- "append"

‎airbyte-integrations/bases/base-normalization/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,5 @@ WORKDIR /airbyte
2828
ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
2929
ENTRYPOINT ["/airbyte/entrypoint.sh"]
3030

31-
LABEL io.airbyte.version=0.2.19
31+
LABEL io.airbyte.version=0.2.20
3232
LABEL io.airbyte.name=airbyte/normalization

‎airbyte-integrations/bases/base-normalization/build.gradle

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,13 +79,18 @@ task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) {
7979
configure buildAirbyteDocker('redshift')
8080
dependsOn assemble
8181
}
82+
task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) {
83+
configure buildAirbyteDocker('tidb')
84+
dependsOn assemble
85+
}
8286

8387
airbyteDocker.dependsOn(airbyteDockerMSSql)
8488
airbyteDocker.dependsOn(airbyteDockerMySql)
8589
airbyteDocker.dependsOn(airbyteDockerOracle)
8690
airbyteDocker.dependsOn(airbyteDockerClickhouse)
8791
airbyteDocker.dependsOn(airbyteDockerSnowflake)
8892
airbyteDocker.dependsOn(airbyteDockerRedshift)
93+
airbyteDocker.dependsOn(airbyteDockerTiDB)
8994

9095
task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) {
9196
module = "pytest"
@@ -100,6 +105,7 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs
100105
dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
101106
dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
102107
dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
108+
dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker'
103109
}
104110

105111
// not really sure what this task does differently from customIntegrationTestPython, but it seems to also run integration tests
@@ -113,6 +119,7 @@ project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-
113119
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker'
114120
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker'
115121
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker'
122+
project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker'
116123

117124
// DATs have some additional tests that exercise normalization code paths,
118125
// so we want to run these in addition to the base-normalization integration tests.

‎airbyte-integrations/bases/base-normalization/dbt-project-template-tidb/dbt_project.yml

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# This file is necessary to install dbt-utils with dbt deps
2+
# the content will be overwritten by the transform function
3+
4+
# Name your package! Package names should contain only lowercase characters
5+
# and underscores. A good package name should reflect your organization"s
6+
# name or the intended use of these models
7+
name: "airbyte_utils"
8+
version: "1.0"
9+
config-version: 2
10+
11+
# This setting configures which "profile" dbt uses for this project. Profiles contain
12+
# database connection information, and should be configured in the ~/.dbt/profiles.yml file
13+
profile: "normalize"
14+
15+
# These configurations specify where dbt should look for different types of files.
16+
# The `model-paths` config, for example, states that source models can be found
17+
# in the "models/" directory. You probably won"t need to change these!
18+
model-paths: ["models"]
19+
docs-paths: ["docs"]
20+
analysis-paths: ["analysis"]
21+
test-paths: ["tests"]
22+
seed-paths: ["data"]
23+
macro-paths: ["macros"]
24+
25+
target-path: "../build" # directory which will store compiled SQL files
26+
log-path: "../logs" # directory which will store DBT logs
27+
packages-install-path: "/dbt" # directory which will store external DBT dependencies
28+
29+
clean-targets: # directories to be removed by `dbt clean`
30+
- "build"
31+
- "dbt_modules"
32+
33+
quoting:
34+
database: true
35+
# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
36+
# all schemas should be unquoted
37+
schema: false
38+
identifier: true
39+
40+
# You can define configurations for models in the `model-paths` directory here.
41+
# Using these configurations, you can enable or disable models, change how they
42+
# are materialized, and more!
43+
models:
44+
airbyte_utils:
45+
+materialized: table
46+
generated:
47+
airbyte_ctes:
48+
+tags: airbyte_internal_cte
49+
+materialized: ephemeral
50+
airbyte_incremental:
51+
+tags: incremental_tables
52+
+materialized: incremental
53+
airbyte_tables:
54+
+tags: normalized_tables
55+
+materialized: table
56+
airbyte_views:
57+
+tags: airbyte_internal_views
58+
+materialized: view
59+
60+
vars:
61+
dbt_utils_dispatch_list: ["airbyte_utils"]

‎airbyte-integrations/bases/base-normalization/dbt-project-template-tidb/packages.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# add dependencies. these will get pulled during the `dbt deps` process.
2+
3+
packages:
4+
- git: "https://github.com/fishtown-analytics/dbt-utils.git"
5+
revision: 0.8.2

‎airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/array.sql

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@
4242
left join joined on _airbyte_{{ stream_name }}_hashid = joined._airbyte_hashid
4343
{%- endmacro %}
4444

45+
{% macro tidb__cross_join_unnest(stream_name, array_col) -%}
46+
left join joined on _airbyte_{{ stream_name }}_hashid = joined._airbyte_hashid
47+
{%- endmacro %}
48+
4549
{% macro redshift__cross_join_unnest(stream_name, array_col) -%}
4650
left join joined on _airbyte_{{ stream_name }}_hashid = joined._airbyte_hashid
4751
{%- endmacro %}
@@ -87,6 +91,10 @@
8791
_airbyte_nested_data
8892
{%- endmacro %}
8993

94+
{% macro tidb__unnested_column_value(column_col) -%}
95+
_airbyte_nested_data
96+
{%- endmacro %}
97+
9098
{% macro oracle__unnested_column_value(column_col) -%}
9199
{{ column_col }}
92100
{%- endmacro %}
@@ -181,3 +189,7 @@ joined as (
181189
where numbers.generated_number <= json_length({{ column_col }})
182190
)
183191
{%- endmacro %}
192+
193+
{% macro tidb__unnest_cte(from_table, stream_name, column_col) -%}
194+
{{ mysql__unnest_cte(from_table, stream_name, column_col) }}
195+
{%- endmacro %}

‎airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,7 @@
2626

2727
concat({{ concat_chunks|join(', ') }}, '')
2828
{%- endmacro %}
29+
30+
{% macro tidb__concat(fields) -%}
31+
concat({{ fields|join(', ') }})
32+
{%- endmacro %}

0 commit comments

Comments
 (0)