Skip to content

Commit b0d0dcd

Browse files
jbfbelljbfbell
authored
oracle_x_norm (#36048)
The testing framework for oracle is not functional and will not be fixed for this release
1 parent 7dc39e8 commit b0d0dcd

File tree

15 files changed

+396
-106
lines changed

15 files changed

+396
-106
lines changed

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ plugins {
44
}
55

66
airbyteJavaConnector {
7-
cdkVersionRequired = '0.2.0'
8-
features = ['db-destinations']
7+
cdkVersionRequired = '0.29.10'
8+
features = ['db-destinations', 's3-destinations', 'typing-deduping']
99
useLocalCdk = false
1010
}
1111

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml

+11-5
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,23 @@ data:
77
connectorSubtype: database
88
connectorType: destination
99
definitionId: 3986776d-2319-4de9-8af8-db14c0996e72
10-
dockerImageTag: 0.2.0
10+
dockerImageTag: 1.0.0
1111
dockerRepository: airbyte/destination-oracle-strict-encrypt
1212
githubIssueLabel: destination-oracle
1313
icon: oracle.svg
1414
license: ELv2
1515
name: Oracle
16-
normalizationConfig:
17-
normalizationIntegrationType: oracle
18-
normalizationRepository: airbyte/normalization-oracle
19-
normalizationTag: 0.4.1
2016
releaseStage: alpha
17+
releases:
18+
breakingChanges:
19+
1.0.0:
20+
upgradeDeadline: "2024-05-01"
21+
message: >
22+
This version removes the option to use "normalization" with Oracle. It also changes
23+
the schema and database of Airbyte's "raw" tables to be compatible with the new
24+
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
25+
format. These changes will likely require updates to downstream dbt / SQL models.
26+
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync.
2127
documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle
2228
supportsDbt: true
2329
tags:

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java

+22-22
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@
1717
import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase;
1818
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
1919
import io.airbyte.cdk.db.jdbc.JdbcUtils;
20+
import io.airbyte.cdk.integrations.base.JavaBaseConstants;
2021
import io.airbyte.cdk.integrations.destination.StandardNameTransformer;
2122
import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest;
2223
import io.airbyte.commons.json.Jsons;
2324
import io.airbyte.commons.string.Strings;
24-
import io.airbyte.integrations.destination.oracle.OracleDestination;
2525
import io.airbyte.integrations.destination.oracle.OracleNameTransformer;
2626
import java.sql.SQLException;
27+
import java.time.Duration;
2728
import java.util.ArrayList;
2829
import java.util.HashSet;
2930
import java.util.List;
@@ -73,7 +74,7 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
7374
return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace)
7475
.stream()
7576
.map(r -> Jsons.deserialize(
76-
r.get(OracleDestination.COLUMN_NAME_DATA.replace("\"", "")).asText()))
77+
r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText()))
7778
.collect(Collectors.toList());
7879
}
7980

@@ -113,16 +114,15 @@ protected List<String> resolveIdentifier(final String identifier) {
113114

114115
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName)
115116
throws SQLException {
116-
final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT);
117-
118-
try (final DSLContext dslContext = getDslContext(config)) {
119-
final List<org.jooq.Record> result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList());
120-
return result
121-
.stream()
122-
.map(r -> r.formatJSON(JSON_FORMAT))
123-
.map(Jsons::deserialize)
124-
.collect(Collectors.toList());
125-
}
117+
final String query =
118+
String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase());
119+
final DSLContext dslContext = getDslContext(config);
120+
final List<org.jooq.Record> result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList());
121+
return result
122+
.stream()
123+
.map(r -> r.formatJSON(JSON_FORMAT))
124+
.map(Jsons::deserialize)
125+
.collect(Collectors.toList());
126126
}
127127

128128
private static Database getDatabase(final DSLContext dslContext) {
@@ -151,15 +151,13 @@ protected void setup(final TestDestinationEnv testEnv, final HashSet<String> TES
151151
db.start();
152152

153153
config = getConfig(db);
154+
final DSLContext dslContext = getDslContext(config);
155+
final Database database = getDatabase(dslContext);
156+
database.query(
157+
ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName)));
158+
database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName)));
154159

155-
try (final DSLContext dslContext = getDslContext(config)) {
156-
final Database database = getDatabase(dslContext);
157-
database.query(
158-
ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName)));
159-
database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName)));
160-
161-
((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName);
162-
}
160+
((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName);
163161
}
164162

165163
@Override
@@ -182,7 +180,8 @@ public void testEncryption() throws SQLException {
182180
config.get(JdbcUtils.PORT_KEY).asInt(),
183181
config.get("sid").asText()),
184182
JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" +
185-
"oracle.net.encryption_types_client=( " + algorithm + " )", ";"));
183+
"oracle.net.encryption_types_client=( " + algorithm + " )", ";"),
184+
Duration.ofMinutes(5));
186185
final JdbcDatabase database = new DefaultJdbcDatabase(dataSource);
187186

188187
final String networkServiceBanner =
@@ -208,7 +207,8 @@ public void testCheckProtocol() throws SQLException {
208207
config.get(JdbcUtils.PORT_KEY).asInt(),
209208
config.get("sid").asText()),
210209
JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" +
211-
"oracle.net.encryption_types_client=( " + algorithm + " )", ";"));
210+
"oracle.net.encryption_types_client=( " + algorithm + " )", ";"),
211+
Duration.ofMinutes(5));
212212
final JdbcDatabase database = new DefaultJdbcDatabase(dataSource);
213213

214214
final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual";

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json

+120
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,126 @@
5959
"examples": ["airbyte"],
6060
"default": "airbyte",
6161
"order": 6
62+
},
63+
"raw_data_schema": {
64+
"type": "string",
65+
"description": "The schema to write raw tables into (default: airbyte_internal)",
66+
"title": "Raw Table Schema Name",
67+
"order": 7
68+
},
69+
"tunnel_method": {
70+
"type": "object",
71+
"title": "SSH Tunnel Method",
72+
"description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.",
73+
"oneOf": [
74+
{
75+
"title": "No Tunnel",
76+
"required": ["tunnel_method"],
77+
"properties": {
78+
"tunnel_method": {
79+
"description": "No ssh tunnel needed to connect to database",
80+
"type": "string",
81+
"const": "NO_TUNNEL",
82+
"order": 0
83+
}
84+
}
85+
},
86+
{
87+
"title": "SSH Key Authentication",
88+
"required": [
89+
"tunnel_method",
90+
"tunnel_host",
91+
"tunnel_port",
92+
"tunnel_user",
93+
"ssh_key"
94+
],
95+
"properties": {
96+
"tunnel_method": {
97+
"description": "Connect through a jump server tunnel host using username and ssh key",
98+
"type": "string",
99+
"const": "SSH_KEY_AUTH",
100+
"order": 0
101+
},
102+
"tunnel_host": {
103+
"title": "SSH Tunnel Jump Server Host",
104+
"description": "Hostname of the jump server host that allows inbound ssh tunnel.",
105+
"type": "string",
106+
"order": 1
107+
},
108+
"tunnel_port": {
109+
"title": "SSH Connection Port",
110+
"description": "Port on the proxy/jump server that accepts inbound ssh connections.",
111+
"type": "integer",
112+
"minimum": 0,
113+
"maximum": 65536,
114+
"default": 22,
115+
"examples": ["22"],
116+
"order": 2
117+
},
118+
"tunnel_user": {
119+
"title": "SSH Login Username",
120+
"description": "OS-level username for logging into the jump server host.",
121+
"type": "string",
122+
"order": 3
123+
},
124+
"ssh_key": {
125+
"title": "SSH Private Key",
126+
"description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )",
127+
"type": "string",
128+
"airbyte_secret": true,
129+
"multiline": true,
130+
"order": 4
131+
}
132+
}
133+
},
134+
{
135+
"title": "Password Authentication",
136+
"required": [
137+
"tunnel_method",
138+
"tunnel_host",
139+
"tunnel_port",
140+
"tunnel_user",
141+
"tunnel_user_password"
142+
],
143+
"properties": {
144+
"tunnel_method": {
145+
"description": "Connect through a jump server tunnel host using username and password authentication",
146+
"type": "string",
147+
"const": "SSH_PASSWORD_AUTH",
148+
"order": 0
149+
},
150+
"tunnel_host": {
151+
"title": "SSH Tunnel Jump Server Host",
152+
"description": "Hostname of the jump server host that allows inbound ssh tunnel.",
153+
"type": "string",
154+
"order": 1
155+
},
156+
"tunnel_port": {
157+
"title": "SSH Connection Port",
158+
"description": "Port on the proxy/jump server that accepts inbound ssh connections.",
159+
"type": "integer",
160+
"minimum": 0,
161+
"maximum": 65536,
162+
"default": 22,
163+
"examples": ["22"],
164+
"order": 2
165+
},
166+
"tunnel_user": {
167+
"title": "SSH Login Username",
168+
"description": "OS-level username for logging into the jump server host",
169+
"type": "string",
170+
"order": 3
171+
},
172+
"tunnel_user_password": {
173+
"title": "Password",
174+
"description": "OS-level password for logging into the jump server host",
175+
"type": "string",
176+
"airbyte_secret": true,
177+
"order": 4
178+
}
179+
}
180+
}
181+
]
62182
}
63183
}
64184
}

‎airbyte-integrations/connectors/destination-oracle/build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ plugins {
44
}
55

66
airbyteJavaConnector {
7-
cdkVersionRequired = '0.2.0'
8-
features = ['db-destinations']
7+
cdkVersionRequired = '0.29.10'
8+
features = ['db-destinations', 's3-destinations', 'typing-deduping']
99
useLocalCdk = false
1010
}
1111

‎airbyte-integrations/connectors/destination-oracle/metadata.yaml

+11-5
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@ data:
22
connectorSubtype: database
33
connectorType: destination
44
definitionId: 3986776d-2319-4de9-8af8-db14c0996e72
5-
dockerImageTag: 0.2.0
5+
dockerImageTag: 1.0.0
66
dockerRepository: airbyte/destination-oracle
77
githubIssueLabel: destination-oracle
88
icon: oracle.svg
99
license: ELv2
1010
name: Oracle
11-
normalizationConfig:
12-
normalizationIntegrationType: oracle
13-
normalizationRepository: airbyte/normalization-oracle
14-
normalizationTag: 0.4.3
1511
registries:
1612
cloud:
1713
dockerRepository: airbyte/destination-oracle-strict-encrypt
@@ -21,6 +17,16 @@ data:
2117
releaseStage: alpha
2218
documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle
2319
supportsDbt: true
20+
releases:
21+
breakingChanges:
22+
1.0.0:
23+
upgradeDeadline: "2024-05-01"
24+
message: >
25+
This version removes the option to use "normalization" with Oracle. It also changes
26+
the schema and database of Airbyte's "raw" tables to be compatible with the new
27+
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
28+
format. These changes will likely require updates to downstream dbt / SQL models.
29+
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync.
2430
tags:
2531
- language:java
2632
ab_internal:

0 commit comments

Comments
 (0)