🎉 Source MySql: Added SSL certificates to MySql Source (#15044)

* updated mysql source specification and added field for root and clients SSL certificates

* added SSL mode for mysql source

* fixed code style

* updated run process timeout

* updated method for create keystore and updated tests

* updated normalization version for postgres destination

* updated normalization version for postgres destination

* added tests for connection with certificates

* updated tests for connection with full certificates and added tests for CA certificate

* updated tests

* updated source-mysql-strict-encrypt and updated versions

* updated code style

* updated doc

* updated specs

* fixed minor remarks

* fixed minor remarks

* updated tests

* fixed remarks and updated specification

* fixed mysql sources connectors version

* added CDC + SSL Certificates tests

* added property for CDC and added tests for test SSL with CDC together

* fixed MySqlStrictEncryptJdbcSourceAcceptanceTest for work with datetime format

* added property for CDC and added tests for test SSL with CDC together

* auto-bump connector version [ci skip]

Co-authored-by: Octavia Squidington III <[email protected]>

Author: 2 people

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -647,7 +647,7 @@
 - name: MySQL
   sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
   dockerRepository: airbyte/source-mysql
-  dockerImageTag: 0.6.2
+  dockerImageTag: 0.6.3
   documentationUrl: https://docs.airbyte.io/integrations/sources/mysql
   icon: mysql.svg
   sourceType: database

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -5977,7 +5977,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-mysql:0.6.2"
+- dockerImage: "airbyte/source-mysql:0.6.3"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql"
     connectionSpecification:
@@ -6035,6 +6035,135 @@
           type: "boolean"
           default: true
           order: 6
+        ssl_mode:
+          title: "SSL modes"
+          description: "SSL connection modes. <li><b>preferred</b> - Automatically\
+            \ attempt SSL connection. If the MySQL server does not support SSL, continue\
+            \ with a regular connection.</li><li><b>required</b> - Always connect\
+            \ with SSL. If the MySQL server doesn’t support SSL, the connection will\
+            \ not be established. Certificate Authority (CA) and Hostname are not\
+            \ verified.</li><li><b>verify-ca</b> - Always connect with SSL. Verifies\
+            \ CA, but allows connection even if Hostname does not match.</li><li><b>Verify\
+            \ Identity</b> - Always connect with SSL. Verify both CA and Hostname.</li></ul>Read\
+            \ more <a href=\"https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-using-ssl.html\"\
+            > in the docs</a>."
+          type: "object"
+          order: 7
+          oneOf:
+          - title: "preferred"
+            description: "Preferred SSL mode."
+            required:
+            - "mode"
+            properties:
+              mode:
+                type: "string"
+                const: "preferred"
+                enum:
+                - "preferred"
+                default: "preferred"
+                order: 0
+          - title: "required"
+            description: "Require SSL mode."
+            required:
+            - "mode"
+            properties:
+              mode:
+                type: "string"
+                const: "required"
+                enum:
+                - "required"
+                default: "required"
+                order: 0
+          - title: "Verify CA"
+            description: "Verify CA SSL mode."
+            required:
+            - "mode"
+            - "ca_certificate"
+            properties:
+              mode:
+                type: "string"
+                const: "verify_ca"
+                enum:
+                - "verify_ca"
+                default: "verify_ca"
+                order: 0
+              ca_certificate:
+                type: "string"
+                title: "CA certificate"
+                description: "CA certificate"
+                airbyte_secret: true
+                multiline: true
+                order: 1
+              client_certificate:
+                type: "string"
+                title: "Client certificate"
+                description: "Client certificate (this is not a required field, but\
+                  \ if you want to use it, you will need to add the <b>Client key</b>\
+                  \ as well)"
+                airbyte_secret: true
+                multiline: true
+                order: 2
+              client_key:
+                type: "string"
+                title: "Client key"
+                description: "Client key (this is not a required field, but if you\
+                  \ want to use it, you will need to add the <b>Client certificate</b>\
+                  \ as well)"
+                airbyte_secret: true
+                multiline: true
+                order: 3
+              client_key_password:
+                type: "string"
+                title: "Client key password (Optional)"
+                description: "Password for keystorage. This field is optional. If\
+                  \ you do not add it - the password will be generated automatically."
+                airbyte_secret: true
+                order: 4
+          - title: "Verify Identity"
+            description: "Verify-full SSL mode."
+            required:
+            - "mode"
+            - "ca_certificate"
+            properties:
+              mode:
+                type: "string"
+                const: "verify_identity"
+                enum:
+                - "verify_identity"
+                default: "verify_identity"
+                order: 0
+              ca_certificate:
+                type: "string"
+                title: "CA certificate"
+                description: "CA certificate"
+                airbyte_secret: true
+                multiline: true
+                order: 1
+              client_certificate:
+                type: "string"
+                title: "Client certificate"
+                description: "Client certificate (this is not a required field, but\
+                  \ if you want to use it, you will need to add the <b>Client key</b>\
+                  \ as well)"
+                airbyte_secret: true
+                multiline: true
+                order: 2
+              client_key:
+                type: "string"
+                title: "Client key"
+                description: "Client key (this is not a required field, but if you\
+                  \ want to use it, you will need to add the <b>Client certificate</b>\
+                  \ as well)"
+                airbyte_secret: true
+                multiline: true
+                order: 3
+              client_key_password:
+                type: "string"
+                title: "Client key password (Optional)"
+                description: "Password for keystorage. This field is optional. If\
+                  \ you do not add it - the password will be generated automatically."
+                airbyte_secret: true
+                order: 4
         replication_method:
           type: "string"
           title: "Replication Method"
@@ -6043,7 +6172,7 @@
             \ but will not be able to represent deletions incrementally. CDC uses\
             \ the Binlog to detect inserts, updates, and deletes. This needs to be\
             \ configured on the source database itself."
-          order: 7
+          order: 8
           default: "STANDARD"
           enum:
           - "STANDARD"

airbyte-db/db-lib/build.gradle

@@ -15,6 +15,7 @@ dependencies {
 
     // Mark as compile only to avoid leaking transitively to connectors
     compileOnly libs.platform.testcontainers.postgresql
+    compileOnly libs.connectors.testcontainers.mysql
 
     // These are required because gradle might be using lower version of Jna from other
     // library transitive dependency. Can be removed if we can figure out which library is the cause.
@@ -25,6 +26,7 @@ dependencies {
     testImplementation project(':airbyte-test-utils')
     testImplementation 'org.apache.commons:commons-lang3:3.11'
     testImplementation libs.platform.testcontainers.postgresql
+    testImplementation libs.connectors.testcontainers.mysql
 
     // Big Query
     implementation('com.google.cloud:google-cloud-bigquery:1.133.1')

airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.db;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import org.testcontainers.containers.MySQLContainer;
+
+public class MySqlUtils {
+
+  @VisibleForTesting
+  public static Certificate getCertificate(final MySQLContainer<?> container,
+                                           final boolean useAllCertificates)
+      throws IOException, InterruptedException {
+    // add root and server certificates to config file
+    container.execInContainer("sh", "-c", "sed -i '31 a ssl' /etc/my.cnf");
+    container.execInContainer("sh", "-c", "sed -i '32 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf");
+    container.execInContainer("sh", "-c", "sed -i '33 a ssl-cert=/var/lib/mysql/server-cert.pem' /etc/my.cnf");
+    container.execInContainer("sh", "-c", "sed -i '34 a ssl-key=/var/lib/mysql/server-key.pem' /etc/my.cnf");
+    container.execInContainer("sh", "-c", "sed -i '35 a require_secure_transport=ON' /etc/my.cnf");
+    // add client certificates to config file
+    if (useAllCertificates) {
+      container.execInContainer("sh", "-c", "sed -i '39 a [client]' /etc/mysql/my.cnf");
+      container.execInContainer("sh", "-c", "sed -i '40 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf");
+      container.execInContainer("sh", "-c", "sed -i '41 a ssl-cert=/var/lib/mysql/client-cert.pem' /etc/my.cnf");
+      container.execInContainer("sh", "-c", "sed -i '42 a ssl-key=/var/lib/mysql/client-key.pem' /etc/my.cnf");
+    }
+    // copy root certificate and client certificates
+    var caCert = container.execInContainer("sh", "-c", "cat /var/lib/mysql/ca.pem").getStdout().trim();
+
+    if (useAllCertificates) {
+      var clientKey = container.execInContainer("sh", "-c", "cat /var/lib/mysql/client-key.pem").getStdout().trim();
+      var clientCert = container.execInContainer("sh", "-c", "cat /var/lib/mysql/client-cert.pem").getStdout().trim();
+      return new Certificate(caCert, clientCert, clientKey);
+    } else {
+      return new Certificate(caCert);
+    }
+  }
+
+  public static class Certificate {
+
+    private final String caCertificate;
+    private final String clientCertificate;
+    private final String clientKey;
+
+    public Certificate(final String caCertificate) {
+      this.caCertificate = caCertificate;
+      this.clientCertificate = null;
+      this.clientKey = null;
+    }
+
+    public Certificate(final String caCertificate, final String clientCertificate, final String clientKey) {
+      this.caCertificate = caCertificate;
+      this.clientCertificate = clientCertificate;
+      this.clientKey = clientKey;
+    }
+
+    public String getCaCertificate() {
+      return caCertificate;
+    }
+
+    public String getClientCertificate() {
+      return clientCertificate;
+    }
+
+    public String getClientKey() {
+      return clientKey;
+    }
+
+  }
+
+}

airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java

@@ -32,6 +32,8 @@ public class JdbcUtils {
   public static final String SSL_MODE_KEY = "ssl_mode";
   public static final String TLS_KEY = "tls";
   public static final String USERNAME_KEY = "username";
+  public static final String MODE_KEY = "mode";
+  public static final String AMPERSAND = "&";
   private static final JdbcSourceOperations defaultSourceOperations = new JdbcSourceOperations();
 
   private static final JSONFormat defaultJSONFormat = new JSONFormat().recordFormat(JSONFormat.RecordFormat.OBJECT);