Simplify Github and Postgres forms #2 (#24255)

* add grouping and collapsing fields to postgres source

* add auth group to github source connector

* revert postgres field order changes and adjust group of schemas field

* inject group into ssh tunnel spec for postgres only, through overloaded methods

* Automated Change

* bump Dockerfile versions and update changelogs

* bump strict encrypt version as well

* fix postgres acceptance test

* fix acceptance test again

* fix all postgres acceptance tests

* add newline

* undo other changes to postgres readme file

* add security group to tunnel_method in expected_spec.json

* bump version of strict encrypt

* manually bump versions in seed files

---------

Co-authored-by: lmossman <[email protected]>

Author: lmossman

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -691,7 +691,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.4.3
+  dockerImageTag: 0.4.4
   documentationUrl: https://docs.airbyte.com/integrations/sources/github
   icon: github.svg
   sourceType: api
@@ -1565,7 +1565,7 @@
 - name: Postgres
   sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750
   dockerRepository: airbyte/source-postgres
-  dockerImageTag: 2.0.7
+  dockerImageTag: 2.0.8
   documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
   icon: postgresql.svg
   sourceType: database

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4938,7 +4938,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.4.3"
+- dockerImage: "airbyte/source-github:0.4.4"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:
@@ -4955,6 +4955,7 @@
           description: "Choose how to authenticate to GitHub"
           type: "object"
           order: 0
+          group: "auth"
           oneOf:
           - type: "object"
             title: "OAuth"
@@ -11911,7 +11912,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-postgres:2.0.7"
+- dockerImage: "airbyte/source-postgres:2.0.8"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"
     connectionSpecification:
@@ -11929,6 +11930,7 @@
           description: "Hostname of the database."
           type: "string"
           order: 0
+          group: "db"
         port:
           title: "Port"
           description: "Port of the database."
@@ -11939,11 +11941,13 @@
           examples:
           - "5432"
           order: 1
+          group: "db"
         database:
           title: "Database Name"
           description: "Name of the database."
           type: "string"
           order: 2
+          group: "db"
         schemas:
           title: "Schemas"
           description: "The list of schemas (case sensitive) to sync from. Defaults\
@@ -11956,17 +11960,21 @@
           default:
           - "public"
           order: 3
+          group: "db"
         username:
           title: "Username"
           description: "Username to access the database."
           type: "string"
           order: 4
+          group: "auth"
         password:
           title: "Password"
           description: "Password associated with the username."
           type: "string"
           airbyte_secret: true
           order: 5
+          group: "auth"
+          always_show: true
         jdbc_url_params:
           description: "Additional properties to pass to the JDBC URL string when\
             \ connecting to the database formatted as 'key=value' pairs separated\
@@ -11976,19 +11984,23 @@
           title: "JDBC URL Parameters (Advanced)"
           type: "string"
           order: 6
+          group: "advanced"
         ssl:
           title: "Connect using SSL"
           description: "Encrypt data using SSL. When activating SSL, please select\
             \ one of the connection modes."
           type: "boolean"
           default: false
           order: 7
+          group: "security"
+          always_show: true
         ssl_mode:
           title: "SSL Modes"
           description: "SSL connection modes. \n  Read more <a href=\"https://jdbc.postgresql.org/documentation/head/ssl-client.html\"\
             > in the docs</a>."
           type: "object"
-          order: 7
+          order: 8
+          group: "security"
           oneOf:
           - title: "disable"
             additionalProperties: true
@@ -12117,7 +12129,8 @@
           type: "object"
           title: "Replication Method"
           description: "Replication method for extracting data from the database."
-          order: 8
+          order: 9
+          group: "advanced"
           oneOf:
           - title: "Standard"
             description: "Standard replication requires no setup on the DB side but\
@@ -12294,6 +12307,14 @@
                 type: "string"
                 airbyte_secret: true
                 order: 4
+          group: "security"
+      groups:
+      - id: "db"
+      - id: "auth"
+      - id: "security"
+        title: "Security"
+      - id: "advanced"
+        title: "Advanced"
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java

@@ -9,18 +9,32 @@
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.protocol.models.v0.ConnectorSpecification;
 import java.io.IOException;
+import java.util.Optional;
 
 public class SshHelpers {
 
   public static ConnectorSpecification getSpecAndInjectSsh() throws IOException {
+    return getSpecAndInjectSsh(Optional.empty());
+  }
+
+  public static ConnectorSpecification getSpecAndInjectSsh(final Optional<String> group) throws IOException {
     final ConnectorSpecification originalSpec = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class);
-    return injectSshIntoSpec(originalSpec);
+    return injectSshIntoSpec(originalSpec, group);
   }
 
   public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecification connectorSpecification) throws IOException {
+    return injectSshIntoSpec(connectorSpecification, Optional.empty());
+  }
+
+  public static ConnectorSpecification injectSshIntoSpec(final ConnectorSpecification connectorSpecification, final Optional<String> group)
+      throws IOException {
     final ConnectorSpecification originalSpec = Jsons.clone(connectorSpecification);
     final ObjectNode propNode = (ObjectNode) originalSpec.getConnectionSpecification().get("properties");
-    propNode.set("tunnel_method", Jsons.deserialize(MoreResources.readResource("ssh-tunnel-spec.json")));
+    final ObjectNode tunnelMethod = (ObjectNode) Jsons.deserialize(MoreResources.readResource("ssh-tunnel-spec.json"));
+    if (group.isPresent()) {
+      tunnelMethod.put("group", group.get());
+    }
+    propNode.set("tunnel_method", tunnelMethod);
     return originalSpec;
   }
 

airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java

@@ -16,6 +16,7 @@
 import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog;
 import io.airbyte.protocol.models.v0.ConnectorSpecification;
 import java.util.List;
+import java.util.Optional;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -25,16 +26,25 @@ public class SshWrappedSource implements Source {
   private final Source delegate;
   private final List<String> hostKey;
   private final List<String> portKey;
+  private final Optional<String> sshGroup;
 
   public SshWrappedSource(final Source delegate, final List<String> hostKey, final List<String> portKey) {
     this.delegate = delegate;
     this.hostKey = hostKey;
     this.portKey = portKey;
+    this.sshGroup = Optional.empty();
+  }
+
+  public SshWrappedSource(final Source delegate, final List<String> hostKey, final List<String> portKey, final String sshGroup) {
+    this.delegate = delegate;
+    this.hostKey = hostKey;
+    this.portKey = portKey;
+    this.sshGroup = Optional.of(sshGroup);
   }
 
   @Override
   public ConnectorSpecification spec() throws Exception {
-    return SshHelpers.injectSshIntoSpec(delegate.spec());
+    return SshHelpers.injectSshIntoSpec(delegate.spec(), sshGroup);
   }
 
   @Override

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.4.3
+LABEL io.airbyte.version=0.4.4
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/source_github/spec.json

@@ -12,6 +12,7 @@
         "description": "Choose how to authenticate to GitHub",
         "type": "object",
         "order": 0,
+        "group": "auth",
         "oneOf": [
           {
             "type": "object",

airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=2.0.7
+LABEL io.airbyte.version=2.0.8
 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt

airbyte-integrations/connectors/source-postgres/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION source-postgres
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=2.0.7
+LABEL io.airbyte.version=2.0.8
 LABEL io.airbyte.name=airbyte/source-postgres

airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java

@@ -117,7 +117,7 @@ public class PostgresSource extends AbstractJdbcSource<PostgresType> implements
   private static final Set<String> INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer");
 
   public static Source sshWrappedSource() {
-    return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY);
+    return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security");
   }
 
   PostgresSource() {

airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json

@@ -10,7 +10,8 @@
         "title": "Host",
         "description": "Hostname of the database.",
         "type": "string",
-        "order": 0
+        "order": 0,
+        "group": "db"
       },
       "port": {
         "title": "Port",
@@ -20,13 +21,15 @@
         "maximum": 65536,
         "default": 5432,
         "examples": ["5432"],
-        "order": 1
+        "order": 1,
+        "group": "db"
       },
       "database": {
         "title": "Database Name",
         "description": "Name of the database.",
         "type": "string",
-        "order": 2
+        "order": 2,
+        "group": "db"
       },
       "schemas": {
         "title": "Schemas",
@@ -38,39 +41,47 @@
         "minItems": 0,
         "uniqueItems": true,
         "default": ["public"],
-        "order": 3
+        "order": 3,
+        "group": "db"
       },
       "username": {
         "title": "Username",
         "description": "Username to access the database.",
         "type": "string",
-        "order": 4
+        "order": 4,
+        "group": "auth"
       },
       "password": {
         "title": "Password",
         "description": "Password associated with the username.",
         "type": "string",
         "airbyte_secret": true,
-        "order": 5
+        "order": 5,
+        "group": "auth",
+        "always_show": true
       },
       "jdbc_url_params": {
         "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about <a href=\"https://jdbc.postgresql.org/documentation/head/connect.html\">JDBC URL parameters</a>.",
         "title": "JDBC URL Parameters (Advanced)",
         "type": "string",
-        "order": 6
+        "order": 6,
+        "group": "advanced"
       },
       "ssl": {
         "title": "Connect using SSL",
         "description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes.",
         "type": "boolean",
         "default": false,
-        "order": 7
+        "order": 7,
+        "group": "security",
+        "always_show": true
       },
       "ssl_mode": {
         "title": "SSL Modes",
         "description": "SSL connection modes. \n  Read more <a href=\"https://jdbc.postgresql.org/documentation/head/ssl-client.html\"> in the docs</a>.",
         "type": "object",
-        "order": 7,
+        "order": 8,
+        "group": "security",
         "oneOf": [
           {
             "title": "disable",
@@ -218,7 +229,8 @@
         "type": "object",
         "title": "Replication Method",
         "description": "Replication method for extracting data from the database.",
-        "order": 8,
+        "order": 9,
+        "group": "advanced",
         "oneOf": [
           {
             "title": "Standard",
@@ -276,14 +288,33 @@
                 "type": "string",
                 "title": "LSN commit behaviour",
                 "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.",
-                "enum": ["While reading Data", "After loading Data in the destination"],
+                "enum": [
+                  "While reading Data",
+                  "After loading Data in the destination"
+                ],
                 "default": "After loading Data in the destination",
                 "order": 6
               }
             }
           }
         ]
       }
-    }
+    },
+    "groups": [
+      {
+        "id": "db"
+      },
+      {
+        "id": "auth"
+      },
+      {
+        "id": "security",
+        "title": "Security"
+      },
+      {
+        "id": "advanced",
+        "title": "Advanced"
+      }
+    ]
   }
 }

airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceAcceptanceTest.java

@@ -0,0 +1,18 @@
+package io.airbyte.integrations.io.airbyte.integration_tests.sources;
+
+import io.airbyte.integrations.base.ssh.SshHelpers;
+import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest;
+import io.airbyte.protocol.models.v0.ConnectorSpecification;
+import java.util.Optional;
+
+public abstract class AbstractPostgresSourceAcceptanceTest extends SourceAcceptanceTest {
+  @Override
+  protected String getImageName() {
+    return "airbyte/source-postgres:dev";
+  }
+
+  @Override
+  protected ConnectorSpecification getSpec() throws Exception {
+    return SshHelpers.getSpecAndInjectSsh(Optional.of("security"));
+  }
+}