Tweak dbt configuration parameters to reasonable values (#9846)

* Tweak dbt configuration parameters to reasonable values (following dbt docs/recommendatins)

* Fix unit tests

* Bumpversion of normalization

Author: ChristopheDuong

airbyte-integrations/bases/base-normalization/Dockerfile

@@ -28,5 +28,5 @@ WORKDIR /airbyte
 ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
 ENTRYPOINT ["/airbyte/entrypoint.sh"]
 
-LABEL io.airbyte.version=0.1.64
+LABEL io.airbyte.version=0.1.65
 LABEL io.airbyte.name=airbyte/normalization

airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py

@@ -133,8 +133,8 @@ def transform_bigquery(config: Dict[str, Any]):
             "project": config["project_id"],
             "dataset": config["dataset_id"],
             "priority": config.get("transformation_priority", "interactive"),
-            "threads": 32,
-            "retries": 1,
+            "threads": 8,
+            "retries": 3,
         }
         if "credentials_json" in config:
             dbt_config["method"] = "service-account-json"
@@ -161,7 +161,7 @@ def transform_postgres(config: Dict[str, Any]):
             "port": config["port"],
             "dbname": config["database"],
             "schema": config["schema"],
-            "threads": 32,
+            "threads": 8,
         }
 
         # if unset, we assume true.
@@ -182,7 +182,7 @@ def transform_redshift(config: Dict[str, Any]):
             "port": config["port"],
             "dbname": config["database"],
             "schema": config["schema"],
-            "threads": 32,
+            "threads": 4,
         }
         return dbt_config
 
@@ -202,9 +202,13 @@ def transform_snowflake(config: Dict[str, Any]):
             "database": config["database"].upper(),
             "warehouse": config["warehouse"].upper(),
             "schema": config["schema"].upper(),
-            "threads": 32,
+            "threads": 5,
             "client_session_keep_alive": False,
             "query_tag": "normalization",
+            "retry_all": True,
+            "retry_on_database_errors": True,
+            "connect_retries": 3,
+            "connect_timeout": 15,
         }
         return dbt_config
 
@@ -259,7 +263,7 @@ def transform_mssql(config: Dict[str, Any]):
             "database": config["database"],
             "user": config["username"],
             "password": config["password"],
-            "threads": 32,
+            "threads": 8,
             # "authentication": "sql",
             # "trusted_connection": True,
         }

airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py

@@ -147,8 +147,8 @@ def test_transform_bigquery(self):
             "priority": "interactive",
             "keyfile_json": {"type": "service_account-json"},
             "location": "EU",
-            "retries": 1,
-            "threads": 32,
+            "retries": 3,
+            "threads": 8,
         }
 
         actual_keyfile = actual_output["keyfile_json"]
@@ -167,8 +167,8 @@ def test_transform_bigquery_no_credentials(self):
             "project": "my_project_id",
             "dataset": "my_dataset_id",
             "priority": "interactive",
-            "retries": 1,
-            "threads": 32,
+            "retries": 3,
+            "threads": 8,
         }
 
         assert expected_output == actual_output
@@ -192,7 +192,7 @@ def test_transform_postgres(self):
             "pass": "password123",
             "port": 5432,
             "schema": "public",
-            "threads": 32,
+            "threads": 8,
             "user": "a user",
         }
 
@@ -225,7 +225,7 @@ def test_transform_postgres_ssh(self):
             "pass": "password123",
             "port": port,
             "schema": "public",
-            "threads": 32,
+            "threads": 8,
             "user": "a user",
         }
 
@@ -252,7 +252,11 @@ def test_transform_snowflake(self):
             "query_tag": "normalization",
             "role": "AIRBYTE_ROLE",
             "schema": "AIRBYTE_SCHEMA",
-            "threads": 32,
+            "threads": 5,
+            "retry_all": True,
+            "retry_on_database_errors": True,
+            "connect_retries": 3,
+            "connect_timeout": 15,
             "type": "snowflake",
             "user": "AIRBYTE_USER",
             "warehouse": "AIRBYTE_WAREHOUSE",
@@ -332,7 +336,7 @@ def test_transform(self):
             "pass": "password123",
             "port": 5432,
             "schema": "public",
-            "threads": 32,
+            "threads": 8,
             "user": "a user",
         }
         actual = TransformConfig().transform(DestinationType.postgres, input)

airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java

@@ -14,7 +14,7 @@
 public class NormalizationRunnerFactory {
 
   public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization";
-  public static final String NORMALIZATION_VERSION = "0.1.64";
+  public static final String NORMALIZATION_VERSION = "0.1.65";
 
   static final Map<String, ImmutablePair<String, DefaultNormalizationRunner.DestinationType>> NORMALIZATION_MAPPING =
       ImmutableMap.<String, ImmutablePair<String, DefaultNormalizationRunner.DestinationType>>builder()

docs/understanding-airbyte/basic-normalization.md

@@ -350,6 +350,8 @@ Therefore, in order to "upgrade" to the desired normalization version, you need
 
 | Airbyte Version | Normalization Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- | :--- |
+| 0.35.13-alpha | 0.1.65 | 2021-01-28 | [\#9846](https://github.com/airbytehq/airbyte/pull/9846) | Tweak dbt multi-thread parameter down |
+| 0.35.12-alpha | 0.1.64 | 2021-01-28 | [\#9793](https://github.com/airbytehq/airbyte/pull/9793) | Support PEM format for ssh-tunnel keys |
 | 0.35.4-alpha | 0.1.63 | 2021-01-07 | [\#9301](https://github.com/airbytehq/airbyte/pull/9301) | Fix Snowflake prefix tables starting with numbers |
 |  | 0.1.62 | 2021-01-07 | [\#9340](https://github.com/airbytehq/airbyte/pull/9340) | Use TCP-port support for clickhouse |
 |  | 0.1.62 | 2021-01-07 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063) | Change Snowflake-specific materialization settings |