JDBC sources: update for sources that not have permissions issues (#10173)

sashaNeshcheret · web-flow · commit 84d1456fcf61 · 2022-02-09T19:23:23.000+02:00
* JDBC sources: update for sources that not have permissions issues

* Postgres, CockroachDB and DB2 sources: updated case when user has none permitted table

* Postgres, CockroachDB and DB2 sources: updated case when user has none permitted table

* Postgres, CockroachDB and DB2 sources: bump versions

* Postgres, CockroachDB and DB2 sources: bump source specs

* Postgres, CockroachDB and DB2 sources: revert changes for postgres-strict-encrypt
diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/447e0381-3780-4b46-bb62-00a4e3c8b8e2.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/447e0381-3780-4b46-bb62-00a4e3c8b8e2.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "447e0381-3780-4b46-bb62-00a4e3c8b8e2",
   "name": "IBM Db2",
   "dockerRepository": "airbyte/source-db2",
-  "dockerImageTag": "0.1.5",
+  "dockerImageTag": "0.1.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/db2",
   "icon": "db2.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9fa5862c-da7c-11eb-8d19-0242ac130003.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9fa5862c-da7c-11eb-8d19-0242ac130003.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "9fa5862c-da7c-11eb-8d19-0242ac130003",
   "name": "Cockroachdb",
   "dockerRepository": "airbyte/source-cockroachdb",
-  "dockerImageTag": "0.1.5",
+  "dockerImageTag": "0.1.6",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/cockroachdb",
   "icon": "cockroachdb.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/decd338e-5647-4c0b-adf4-da0e75f5a750.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",
   "name": "Postgres",
   "dockerRepository": "airbyte/source-postgres",
-  "dockerImageTag": "0.4.4",
+  "dockerImageTag": "0.4.5",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres",
   "icon": "postgresql.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -140,7 +140,7 @@
 - name: Cockroachdb
   sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   dockerRepository: airbyte/source-cockroachdb
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb
   icon: cockroachdb.svg
   sourceType: database
@@ -321,7 +321,7 @@
 - name: IBM Db2
   sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2
   dockerRepository: airbyte/source-db2
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/db2
   icon: db2.svg
   sourceType: database
@@ -578,7 +578,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 0.4.4
+  dockerImageTag: 0.4.5
   documentationUrl: https://docs.airbyte.io/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database
diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -1205,7 +1205,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-cockroachdb:0.1.5"
+- dockerImage: "airbyte/source-cockroachdb:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/cockroachdb"
     connectionSpecification:
@@ -3171,7 +3171,7 @@
         - - "client_secret"
         oauthFlowOutputParameters:
         - - "refresh_token"
-- dockerImage: "airbyte/source-db2:0.1.5"
+- dockerImage: "airbyte/source-db2:0.1.6"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/db2"
     connectionSpecification:
@@ -5942,7 +5942,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:0.4.4"
+- dockerImage: "airbyte/source-postgres:0.4.5"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-cockroachdb
diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java
@@ -105,6 +105,11 @@ public Set<JdbcPrivilegeDto> getPrivilegesTableForCurrentUser(final JdbcDatabase
         .collect(Collectors.toSet());
   }
 
+  @Override
+  protected boolean isNotInternalSchema(JsonNode jsonNode, Set<String> internalSchemas) {
+    return false;
+  }
+
   private CheckedFunction<Connection, PreparedStatement, SQLException> getPrivileges(JdbcDatabase database) {
     return connection -> {
       final PreparedStatement ps = connection.prepareStatement(
diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt
diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-db2
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-db2
diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java
@@ -95,6 +95,11 @@ public Set<JdbcPrivilegeDto> getPrivilegesTableForCurrentUser(final JdbcDatabase
         .collect(Collectors.toSet());
   }
 
+  @Override
+  protected boolean isNotInternalSchema(JsonNode jsonNode, Set<String> internalSchemas) {
+    return false;
+  }
+
   private CheckedFunction<Connection, PreparedStatement, SQLException> getPrivileges() {
     return connection -> connection.prepareStatement(
         "SELECT DISTINCT OBJECTNAME, OBJECTSCHEMA FROM SYSIBMADM.PRIVILEGES WHERE OBJECTTYPE = 'TABLE' AND PRIVILEGE = 'SELECT' AND AUTHID = SESSION_USER");
diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java
@@ -146,11 +146,11 @@ protected List<TableInfo<CommonField<Datatype>>> discoverInternal(final JdbcData
         .collect(Collectors.toList());
   }
 
-  private Predicate<JsonNode> excludeNotAccessibleTables(final Set<String> internalSchemas,
+  protected Predicate<JsonNode> excludeNotAccessibleTables(final Set<String> internalSchemas,
                                                          final Set<JdbcPrivilegeDto> tablesWithSelectGrantPrivilege) {
     return jsonNode -> {
       if (tablesWithSelectGrantPrivilege.isEmpty()) {
-        return false;
+        return isNotInternalSchema(jsonNode, internalSchemas);
       }
       return tablesWithSelectGrantPrivilege.stream()
           .anyMatch(e -> e.getSchemaName().equals(jsonNode.get(INTERNAL_SCHEMA_NAME).asText()))
@@ -160,6 +160,11 @@ private Predicate<JsonNode> excludeNotAccessibleTables(final Set<String> interna
     };
   }
 
+  // needs to override isNotInternalSchema for connectors that override getPrivilegesTableForCurrentUser()
+  protected boolean isNotInternalSchema(JsonNode jsonNode, Set<String> internalSchemas) {
+    return !internalSchemas.contains(jsonNode.get(INTERNAL_SCHEMA_NAME).asText());
+  }
+
   /**
    * @param resultSet Description of a column available in the table catalog.
    * @return Essential information about a column to determine which table it belongs to and its type.
diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile
@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.4.4
+LABEL io.airbyte.version=0.4.5
 LABEL io.airbyte.name=airbyte/source-postgres
diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java
@@ -323,6 +323,11 @@ public Set<JdbcPrivilegeDto> getPrivilegesTableForCurrentUser(final JdbcDatabase
         .collect(toSet());
   }
 
+  @Override
+  protected boolean isNotInternalSchema(JsonNode jsonNode, Set<String> internalSchemas) {
+    return false;
+  }
+
   /*
    * Set all streams that do have incremental to sourceDefined, so that the user cannot set or
    * override a cursor field.
diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md
@@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte.
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table |
 | 0.1.5 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery |
 | 0.1.4 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY |
 | 0.1.3 | 2021-10-10 | [7819](https://github.com/airbytehq/airbyte/pull/7819) | Fixed Datatype errors during Cockroach DB parsing |
@@ -104,6 +105,7 @@ Your database user should now be ready for use with Airbyte.
 
 | Version | Date | Pull Request | Subject |
 |:--------| :--- | :--- | :--- |
+| 0.1.3   | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table |
 | 0.1.2   | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery |
 | 0.1.1   | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY |
 | 0.1.0   | 2021-11-23 | [7457](https://github.com/airbytehq/airbyte/pull/7457) | CockroachDb source: Add only encrypted version for the connector |
diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md
@@ -62,6 +62,7 @@ You can also enter your own password for the keystore, but if you don't, the pas
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table |
 | 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables |
 | 0.1.4 | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. |
 | 0.1.3 | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation |
diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md
@@ -270,6 +270,7 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp
 
 | Version | Date       | Pull Request                                           | Subject                                                                                                         |
 |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 0.4.5   | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table                                                                       |
 | 0.4.4   | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807) | Update connector fields title/description                                                                       |
 | 0.4.3   | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554) | Allow handling of java sql date in CDC                                                                          |
 | 0.4.2   | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360) | Added schema selection                                                                                          |

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`"sourceDefinitionId": "447e0381-3780-4b46-bb62-00a4e3c8b8e2",`
`3`	`3`	`"name": "IBM Db2",`
`4`	`4`	`"dockerRepository": "airbyte/source-db2",`
`5`		`- "dockerImageTag": "0.1.5",`
	`5`	`+ "dockerImageTag": "0.1.6",`
`6`	`6`	`"documentationUrl": "https://docs.airbyte.io/integrations/sources/db2",`
`7`	`7`	`"icon": "db2.svg"`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`"sourceDefinitionId": "9fa5862c-da7c-11eb-8d19-0242ac130003",`
`3`	`3`	`"name": "Cockroachdb",`
`4`	`4`	`"dockerRepository": "airbyte/source-cockroachdb",`
`5`		`- "dockerImageTag": "0.1.5",`
	`5`	`+ "dockerImageTag": "0.1.6",`
`6`	`6`	`"documentationUrl": "https://docs.airbyte.io/integrations/sources/cockroachdb",`
`7`	`7`	`"icon": "cockroachdb.svg"`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`	`"sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750",`
`3`	`3`	`"name": "Postgres",`
`4`	`4`	`"dockerRepository": "airbyte/source-postgres",`
`5`		`- "dockerImageTag": "0.4.4",`
	`5`	`+ "dockerImageTag": "0.4.5",`
`6`	`6`	`"documentationUrl": "https://docs.airbyte.io/integrations/sources/postgres",`
`7`	`7`	`"icon": "postgresql.svg"`
`8`	`8`	`}`