diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json index aa2f009fdd11c..d637baaddf031 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b39a7370-74c3-45a6-ac3a-380d48520a83.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", "name": "Oracle DB", "dockerRepository": "airbyte/source-oracle", - "dockerImageTag": "0.3.7", + "dockerImageTag": "0.3.8", "documentationUrl": "https://docs.airbyte.io/integrations/sources/oracle" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 96565960fe583..586e3d25186dd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -332,7 +332,7 @@ - sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 name: Oracle DB dockerRepository: airbyte/source-oracle - dockerImageTag: 0.3.7 + dockerImageTag: 0.3.8 documentationUrl: https://docs.airbyte.io/integrations/sources/oracle sourceType: database - sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071 diff --git a/airbyte-integrations/connectors/source-oracle/BOOTSTRAP.md b/airbyte-integrations/connectors/source-oracle/BOOTSTRAP.md new file mode 100644 index 0000000000000..c65fc81a52132 --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle/BOOTSTRAP.md @@ -0,0 +1,10 @@ +# Oracle Source +The Oracle source connector allows syncing the data from the Oracle DB. The current source connector supports Oracle 11g or above. +The connector uses *ojdbc8* driver underneath to establish the connection. The Oracle source does not alter the schema present in your database. + +### Important details +Connector works with `useFetchSizeWithLongColumn=true` property, which required to select the data from `LONG` or `LONG RAW` type columns. +Oracle recommends avoiding LONG and LONG RAW columns. Use LOB instead. They are included in Oracle only for legacy reasons. +THIS IS A THIN ONLY PROPERTY. IT SHOULD NOT BE USED WITH ANY OTHER DRIVERS. + +See [this](https://docs.airbyte.io/integrations/sources/oracle) link for the nuances about the connector. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 9f55b4a11deb2..261f885cda529 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.7 +LABEL io.airbyte.version=0.3.8 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-oracle/README.md b/airbyte-integrations/connectors/source-oracle/README.md new file mode 100644 index 0000000000000..a3a7ebd15aa84 --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle/README.md @@ -0,0 +1,33 @@ +# Oracle Source + +## Documentation +This is the repository for the Oracle only source connector in Java. +For information about how to use this connector within Airbyte, see [User Documentation](https://docs.airbyte.io/integrations/sources/oracle) + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-oracle:build +``` + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-oracle:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +## Testing +We use `JUnit` for Java tests. + +### Test Configuration +#### Acceptance Tests +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-oracle:integrationTest +``` diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index 72592930913b2..8ec53940ec78d 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -55,6 +55,15 @@ public static Source sshWrappedSource() { public JsonNode toDatabaseConfig(final JsonNode config) { final List additionalParameters = new ArrayList<>(); + /* + The property useFetchSizeWithLongColumn required to select LONG or LONG RAW columns. + Oracle recommends avoiding LONG and LONG RAW columns. Use LOB instead. They are included in Oracle only for legacy reasons. + THIS IS A THIN ONLY PROPERTY. IT SHOULD NOT BE USED WITH ANY OTHER DRIVERS. + See https://docs.oracle.com/cd/E11882_01/appdev.112/e13995/oracle/jdbc/OracleDriver.html + https://docs.oracle.com/cd/B19306_01/java.102/b14355/jstreams.htm#i1014085 + */ + additionalParameters.add("oracle.jdbc.useFetchSizeWithLongColumn=true"); + final Protocol protocol = config.has("encryption") ? obtainConnectionProtocol(config.get("encryption"), additionalParameters) : Protocol.TCP; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 498b991200e91..5e2c8aea1dadd 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -174,7 +174,7 @@ protected void initTests() { TestDataHolder.builder() .sourceType("BINARY_DOUBLE") .airbyteType(JsonSchemaPrimitive.NUMBER) - .addInsertValues("126.45d", "2.22507485850720E-308", "TO_BINARY_DOUBLE('1.79769313486231E+308')", "BINARY_DOUBLE_INFINITY") + .addInsertValues("126.45d", "2.22507485850720E-308", "1.79769313486231E+308d", "BINARY_DOUBLE_INFINITY") .addExpectedValues("126.45", "0.0", "1.79769313486231E308", "Infinity") .build()); @@ -266,8 +266,8 @@ protected void initTests() { .sourceType("LONG") .airbyteType(JsonSchemaPrimitive.STRING) .fullSourceDataType("LONG RAW") - // @TODO stream fails when reading data back - // .addInsertValues("utl_raw.cast_to_raw('some content here')", "null") + .addInsertValues("utl_raw.cast_to_raw('some content here')", "null") + .addExpectedValues("c29tZSBjb250ZW50IGhlcmU=", null) .build()); addDataTypeTestData( diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 30b67a5c6eb85..e99f4a9360bbe 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,6 +132,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.3.8 | 2021-10-13 | [7125](https://github.com/airbytehq/airbyte/pull/7125) | Fix incorrect handling of LONG RAW data type | | 0.3.7 | 2021-10-01 | [6616](https://github.com/airbytehq/airbyte/pull/6616) | Added network encryption options | | 0.3.6 | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps | | 0.3.5 | 2021-09-22 | [6356](https://github.com/airbytehq/airbyte/pull/6356) | Added option to connect to DB via SSH. |