Skip to content

Commit 9d4d80d

Browse files
jbfbelljbfbell
committed
fmt
1 parent a895b20 commit 9d4d80d

File tree

17 files changed

+349
-93
lines changed

17 files changed

+349
-93
lines changed

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ plugins {
44
}
55

66
airbyteJavaConnector {
7-
cdkVersionRequired = '0.2.0'
8-
features = ['db-destinations']
9-
useLocalCdk = false
7+
cdkVersionRequired = '0.24.1'
8+
features = ['db-destinations', 's3-destinations', 'typing-deduping']
9+
useLocalCdk = true
1010
}
1111

1212
//remove once upgrading the CDK version to 0.4.x or later

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/gradle.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
junitMethodExecutionTimeout = 30 m

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,17 +7,23 @@ data:
77
connectorSubtype: database
88
connectorType: destination
99
definitionId: 3986776d-2319-4de9-8af8-db14c0996e72
10-
dockerImageTag: 0.2.0
10+
dockerImageTag: 1.0.0
1111
dockerRepository: airbyte/destination-oracle-strict-encrypt
1212
githubIssueLabel: destination-oracle
1313
icon: oracle.svg
1414
license: ELv2
1515
name: Oracle
16-
normalizationConfig:
17-
normalizationIntegrationType: oracle
18-
normalizationRepository: airbyte/normalization-oracle
19-
normalizationTag: 0.4.1
2016
releaseStage: alpha
17+
releases:
18+
breakingChanges:
19+
1.0.0:
20+
upgradeDeadline: "2024-03-15"
21+
message: >
22+
This version removes the option to use "normalization" with Oracle. It also changes
23+
the schema and database of Airbyte's "raw" tables to be compatible with the new
24+
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
25+
format. These changes will likely require updates to downstream dbt / SQL models.
26+
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync.
2127
documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle
2228
supportsDbt: true
2329
tags:

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717
import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase;
1818
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
1919
import io.airbyte.cdk.db.jdbc.JdbcUtils;
20+
import io.airbyte.cdk.integrations.base.JavaBaseConstants;
2021
import io.airbyte.cdk.integrations.destination.StandardNameTransformer;
2122
import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest;
2223
import io.airbyte.commons.json.Jsons;
2324
import io.airbyte.commons.string.Strings;
24-
import io.airbyte.integrations.destination.oracle.OracleDestination;
2525
import io.airbyte.integrations.destination.oracle.OracleNameTransformer;
2626
import java.sql.SQLException;
2727
import java.util.ArrayList;
@@ -73,7 +73,7 @@ protected List<JsonNode> retrieveRecords(final TestDestinationEnv env,
7373
return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace)
7474
.stream()
7575
.map(r -> Jsons.deserialize(
76-
r.get(OracleDestination.COLUMN_NAME_DATA.replace("\"", "")).asText()))
76+
r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText()))
7777
.collect(Collectors.toList());
7878
}
7979

@@ -113,16 +113,15 @@ protected List<String> resolveIdentifier(final String identifier) {
113113

114114
private List<JsonNode> retrieveRecordsFromTable(final String tableName, final String schemaName)
115115
throws SQLException {
116-
final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT);
117-
118-
try (final DSLContext dslContext = getDslContext(config)) {
119-
final List<org.jooq.Record> result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList());
120-
return result
121-
.stream()
122-
.map(r -> r.formatJSON(JSON_FORMAT))
123-
.map(Jsons::deserialize)
124-
.collect(Collectors.toList());
125-
}
116+
final String query =
117+
String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase());
118+
final DSLContext dslContext = getDslContext(config);
119+
final List<org.jooq.Record> result = getDatabase(dslContext).query(ctx -> ctx.fetch(query).stream().toList());
120+
return result
121+
.stream()
122+
.map(r -> r.formatJSON(JSON_FORMAT))
123+
.map(Jsons::deserialize)
124+
.collect(Collectors.toList());
126125
}
127126

128127
private static Database getDatabase(final DSLContext dslContext) {
@@ -151,15 +150,13 @@ protected void setup(final TestDestinationEnv testEnv, final HashSet<String> TES
151150
db.start();
152151

153152
config = getConfig(db);
153+
final DSLContext dslContext = getDslContext(config);
154+
final Database database = getDatabase(dslContext);
155+
database.query(
156+
ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName)));
157+
database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName)));
154158

155-
try (final DSLContext dslContext = getDslContext(config)) {
156-
final Database database = getDatabase(dslContext);
157-
database.query(
158-
ctx -> ctx.fetch(String.format("CREATE USER %s IDENTIFIED BY %s", schemaName, schemaName)));
159-
database.query(ctx -> ctx.fetch(String.format("GRANT ALL PRIVILEGES TO %s", schemaName)));
160-
161-
((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName);
162-
}
159+
((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, dbName);
163160
}
164161

165162
@Override

‎airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/resources/expected_spec.json

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,126 @@
5959
"examples": ["airbyte"],
6060
"default": "airbyte",
6161
"order": 6
62+
},
63+
"raw_data_schema": {
64+
"type": "string",
65+
"description": "The schema to write raw tables into (default: airbyte_internal)",
66+
"title": "Raw Table Schema Name",
67+
"order": 7
68+
},
69+
"tunnel_method": {
70+
"type": "object",
71+
"title": "SSH Tunnel Method",
72+
"description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.",
73+
"oneOf": [
74+
{
75+
"title": "No Tunnel",
76+
"required": ["tunnel_method"],
77+
"properties": {
78+
"tunnel_method": {
79+
"description": "No ssh tunnel needed to connect to database",
80+
"type": "string",
81+
"const": "NO_TUNNEL",
82+
"order": 0
83+
}
84+
}
85+
},
86+
{
87+
"title": "SSH Key Authentication",
88+
"required": [
89+
"tunnel_method",
90+
"tunnel_host",
91+
"tunnel_port",
92+
"tunnel_user",
93+
"ssh_key"
94+
],
95+
"properties": {
96+
"tunnel_method": {
97+
"description": "Connect through a jump server tunnel host using username and ssh key",
98+
"type": "string",
99+
"const": "SSH_KEY_AUTH",
100+
"order": 0
101+
},
102+
"tunnel_host": {
103+
"title": "SSH Tunnel Jump Server Host",
104+
"description": "Hostname of the jump server host that allows inbound ssh tunnel.",
105+
"type": "string",
106+
"order": 1
107+
},
108+
"tunnel_port": {
109+
"title": "SSH Connection Port",
110+
"description": "Port on the proxy/jump server that accepts inbound ssh connections.",
111+
"type": "integer",
112+
"minimum": 0,
113+
"maximum": 65536,
114+
"default": 22,
115+
"examples": ["22"],
116+
"order": 2
117+
},
118+
"tunnel_user": {
119+
"title": "SSH Login Username",
120+
"description": "OS-level username for logging into the jump server host.",
121+
"type": "string",
122+
"order": 3
123+
},
124+
"ssh_key": {
125+
"title": "SSH Private Key",
126+
"description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )",
127+
"type": "string",
128+
"airbyte_secret": true,
129+
"multiline": true,
130+
"order": 4
131+
}
132+
}
133+
},
134+
{
135+
"title": "Password Authentication",
136+
"required": [
137+
"tunnel_method",
138+
"tunnel_host",
139+
"tunnel_port",
140+
"tunnel_user",
141+
"tunnel_user_password"
142+
],
143+
"properties": {
144+
"tunnel_method": {
145+
"description": "Connect through a jump server tunnel host using username and password authentication",
146+
"type": "string",
147+
"const": "SSH_PASSWORD_AUTH",
148+
"order": 0
149+
},
150+
"tunnel_host": {
151+
"title": "SSH Tunnel Jump Server Host",
152+
"description": "Hostname of the jump server host that allows inbound ssh tunnel.",
153+
"type": "string",
154+
"order": 1
155+
},
156+
"tunnel_port": {
157+
"title": "SSH Connection Port",
158+
"description": "Port on the proxy/jump server that accepts inbound ssh connections.",
159+
"type": "integer",
160+
"minimum": 0,
161+
"maximum": 65536,
162+
"default": 22,
163+
"examples": ["22"],
164+
"order": 2
165+
},
166+
"tunnel_user": {
167+
"title": "SSH Login Username",
168+
"description": "OS-level username for logging into the jump server host",
169+
"type": "string",
170+
"order": 3
171+
},
172+
"tunnel_user_password": {
173+
"title": "Password",
174+
"description": "OS-level password for logging into the jump server host",
175+
"type": "string",
176+
"airbyte_secret": true,
177+
"order": 4
178+
}
179+
}
180+
}
181+
]
62182
}
63183
}
64184
}

‎airbyte-integrations/connectors/destination-oracle/build.gradle

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ plugins {
44
}
55

66
airbyteJavaConnector {
7-
cdkVersionRequired = '0.2.0'
8-
features = ['db-destinations']
9-
useLocalCdk = false
7+
cdkVersionRequired = '0.24.1'
8+
features = ['db-destinations', 's3-destinations', 'typing-deduping']
9+
useLocalCdk = true
1010
}
1111

1212
//remove once upgrading the CDK version to 0.4.x or later

‎airbyte-integrations/connectors/destination-oracle/gradle.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
junitMethodExecutionTimeout = 30 m

‎airbyte-integrations/connectors/destination-oracle/metadata.yaml

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@ data:
22
connectorSubtype: database
33
connectorType: destination
44
definitionId: 3986776d-2319-4de9-8af8-db14c0996e72
5-
dockerImageTag: 0.2.0
5+
dockerImageTag: 1.0.0
66
dockerRepository: airbyte/destination-oracle
77
githubIssueLabel: destination-oracle
88
icon: oracle.svg
99
license: ELv2
1010
name: Oracle
11-
normalizationConfig:
12-
normalizationIntegrationType: oracle
13-
normalizationRepository: airbyte/normalization-oracle
14-
normalizationTag: 0.4.3
1511
registries:
1612
cloud:
1713
dockerRepository: airbyte/destination-oracle-strict-encrypt
@@ -21,6 +17,16 @@ data:
2117
releaseStage: alpha
2218
documentationUrl: https://docs.airbyte.com/integrations/destinations/oracle
2319
supportsDbt: true
20+
releases:
21+
breakingChanges:
22+
1.0.0:
23+
upgradeDeadline: "2024-03-15"
24+
message: >
25+
This version removes the option to use "normalization" with Oracle. It also changes
26+
the schema and database of Airbyte's "raw" tables to be compatible with the new
27+
[Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2)
28+
format. These changes will likely require updates to downstream dbt / SQL models.
29+
Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync.
2430
tags:
2531
- language:java
2632
ab_internal:

‎airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,30 @@
77
import com.fasterxml.jackson.databind.JsonNode;
88
import com.google.common.collect.ImmutableMap;
99
import io.airbyte.cdk.db.factory.DatabaseDriver;
10+
import io.airbyte.cdk.db.jdbc.JdbcDatabase;
1011
import io.airbyte.cdk.db.jdbc.JdbcUtils;
1112
import io.airbyte.cdk.integrations.base.Destination;
1213
import io.airbyte.cdk.integrations.base.IntegrationRunner;
13-
import io.airbyte.cdk.integrations.base.JavaBaseConstants;
1414
import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination;
1515
import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination;
16+
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler;
17+
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator;
18+
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.NoOpJdbcDestinationHandler;
19+
import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.RawOnlySqlGenerator;
1620
import io.airbyte.commons.json.Jsons;
21+
import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler;
22+
import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator;
23+
import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration;
24+
import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState;
1725
import java.io.IOException;
1826
import java.io.PrintWriter;
1927
import java.nio.charset.StandardCharsets;
2028
import java.util.HashMap;
29+
import java.util.List;
2130
import java.util.Map;
2231
import java.util.concurrent.TimeUnit;
2332
import org.apache.commons.lang3.RandomStringUtils;
33+
import org.jooq.SQLDialect;
2434
import org.slf4j.Logger;
2535
import org.slf4j.LoggerFactory;
2636

@@ -29,13 +39,6 @@ public class OracleDestination extends AbstractJdbcDestination implements Destin
2939
private static final Logger LOGGER = LoggerFactory.getLogger(OracleDestination.class);
3040
public static final String DRIVER_CLASS = DatabaseDriver.ORACLE.getDriverClassName();
3141

32-
public static final String COLUMN_NAME_AB_ID =
33-
"\"" + JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase() + "\"";
34-
public static final String COLUMN_NAME_DATA =
35-
"\"" + JavaBaseConstants.COLUMN_NAME_DATA.toUpperCase() + "\"";
36-
public static final String COLUMN_NAME_EMITTED_AT =
37-
"\"" + JavaBaseConstants.COLUMN_NAME_EMITTED_AT.toUpperCase() + "\"";
38-
3942
protected static final String KEY_STORE_FILE_PATH = "clientkeystore.jks";
4043
private static final String KEY_STORE_PASS = RandomStringUtils.randomAlphanumeric(8);
4144
public static final String ENCRYPTION_METHOD_KEY = "encryption_method";
@@ -134,6 +137,36 @@ private static void tryConvertAndImportCertificate(final String certificate) {
134137
}
135138
}
136139

140+
@Override
141+
public boolean isV2Destination() {
142+
return true;
143+
}
144+
145+
@Override
146+
protected boolean shouldAlwaysDisableTypeDedupe() {
147+
return true;
148+
}
149+
150+
@Override
151+
protected JdbcSqlGenerator getSqlGenerator() {
152+
return new RawOnlySqlGenerator(new OracleNameTransformer());
153+
}
154+
155+
@Override
156+
protected JdbcDestinationHandler<? extends MinimumDestinationState> getDestinationHandler(final String databaseName,
157+
final JdbcDatabase database,
158+
final String rawTableSchema) {
159+
return new NoOpJdbcDestinationHandler<>(databaseName, database, rawTableSchema, SQLDialect.DEFAULT);
160+
}
161+
162+
@Override
163+
protected List<Migration> getMigrations(final JdbcDatabase database,
164+
final String databaseName,
165+
final SqlGenerator sqlGenerator,
166+
final DestinationHandler destinationHandler) {
167+
return List.of();
168+
}
169+
137170
private static void convertAndImportCertificate(final String certificate)
138171
throws IOException, InterruptedException {
139172
final Runtime run = Runtime.getRuntime();

‎airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public String applyDefaultCase(final String input) {
1717
}
1818

1919
@Override
20+
@Deprecated
2021
public String getRawTableName(final String streamName) {
2122
return convertStreamName("airbyte_raw_" + streamName);
2223
}

0 commit comments

Comments
 (0)