BigQueryDestinationConfig to include merge and Append-only as configure write modes (#11116) (#745)

[upstream:0f42b5700ef7a6f8639231e8a56ab073ebcb7046]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

datastream_stream_bigquery/main.tf

@@ -106,6 +106,7 @@ resource "google_datastream_stream" "default" {
                     kms_key_name = "bigquery-kms-name-${local.name_suffix}"
                 }
             }
+            merge {}
         }
     }
 

datastream_stream_bigquery_append_only/backing_file.tf

@@ -0,0 +1,15 @@
+# This file has some scaffolding to make sure that names are unique and that
+# a region and zone are selected when you try to create your Terraform resources.
+
+locals {
+  name_suffix = "${random_pet.suffix.id}"
+}
+
+resource "random_pet" "suffix" {
+  length = 2
+}
+
+provider "google" {
+  region = "us-central1"
+  zone   = "us-central1-c"
+}

datastream_stream_bigquery_append_only/main.tf

@@ -0,0 +1,102 @@
+data "google_project" "project" {
+}
+
+resource "google_sql_database_instance" "instance" {
+    name             = "my-instance-${local.name_suffix}"
+    database_version = "MYSQL_8_0"
+    region           = "us-central1"
+    settings {
+        tier = "db-f1-micro"
+        backup_configuration {
+            enabled            = true
+            binary_log_enabled = true
+        }
+
+        ip_configuration {
+
+            // Datastream IPs will vary by region.
+            authorized_networks {
+                value = "34.71.242.81"
+            }
+
+            authorized_networks {
+                value = "34.72.28.29"
+            }
+
+            authorized_networks {
+                value = "34.67.6.157"
+            }
+
+            authorized_networks {
+                value = "34.67.234.134"
+            }
+
+            authorized_networks {
+                value = "34.72.239.218"
+            }
+        }
+    }
+
+    deletion_protection  = false
+}
+
+resource "google_sql_database" "db" {
+    instance = google_sql_database_instance.instance.name
+    name     = "db"
+}
+
+resource "random_password" "pwd" {
+    length = 16
+    special = false
+}
+
+resource "google_sql_user" "user" {
+    name     = "user"
+    instance = google_sql_database_instance.instance.name
+    host     = "%"
+    password = random_password.pwd.result
+}
+
+resource "google_datastream_connection_profile" "source_connection_profile" {
+    display_name          = "Source connection profile"
+    location              = "us-central1"
+    connection_profile_id = "source-profile-${local.name_suffix}"
+
+    mysql_profile {
+        hostname = google_sql_database_instance.instance.public_ip_address
+        username = google_sql_user.user.name
+        password = google_sql_user.user.password
+    }
+}
+
+resource "google_datastream_connection_profile" "destination_connection_profile" {
+    display_name          = "Connection profile"
+    location              = "us-central1"
+    connection_profile_id = "destination-profile-${local.name_suffix}"
+
+    bigquery_profile {}
+}
+
+resource "google_datastream_stream" "default" {
+    stream_id = "my-stream-${local.name_suffix}"
+    location = "us-central1"
+    display_name = "my stream"
+    source_config {
+        source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
+        mysql_source_config {}
+    }
+    destination_config {
+        destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id
+        bigquery_destination_config {
+            source_hierarchy_datasets {
+                dataset_template {
+                    location = "us-central1"
+                }
+            }
+            append_only {}
+        }
+    }
+
+    backfill_none {
+    }
+}

datastream_stream_bigquery_append_only/motd

@@ -0,0 +1,7 @@
+===
+
+These examples use real resources that will be billed to the
+Google Cloud Platform project you use - so make sure that you
+run "terraform destroy" before quitting!
+
+===

datastream_stream_bigquery_append_only/tutorial.md

@@ -0,0 +1,79 @@
+# Datastream Stream Bigquery Append Only - Terraform
+
+## Setup
+
+<walkthrough-author name="[email protected]" analyticsId="UA-125550242-1" tutorialName="datastream_stream_bigquery_append_only" repositoryUrl="https://github.com/terraform-google-modules/docs-examples"></walkthrough-author>
+
+Welcome to Terraform in Google Cloud Shell! We need you to let us know what project you'd like to use with Terraform.
+
+<walkthrough-project-billing-setup></walkthrough-project-billing-setup>
+
+Terraform provisions real GCP resources, so anything you create in this session will be billed against this project.
+
+## Terraforming!
+
+Let's use {{project-id}} with Terraform! Click the Cloud Shell icon below to copy the command
+to your shell, and then run it from the shell by pressing Enter/Return. Terraform will pick up
+the project name from the environment variable.
+
+```bash
+export GOOGLE_CLOUD_PROJECT={{project-id}}
+```
+
+After that, let's get Terraform started. Run the following to pull in the providers.
+
+```bash
+terraform init
+```
+
+With the providers downloaded and a project set, you're ready to use Terraform. Go ahead!
+
+```bash
+terraform apply
+```
+
+Terraform will show you what it plans to do, and prompt you to accept. Type "yes" to accept the plan.
+
+```bash
+yes
+```
+
+
+## Post-Apply
+
+### Editing your config
+
+Now you've provisioned your resources in GCP! If you run a "plan", you should see no changes needed.
+
+```bash
+terraform plan
+```
+
+So let's make a change! Try editing a number, or appending a value to the name in the editor. Then,
+run a 'plan' again.
+
+```bash
+terraform plan
+```
+
+Afterwards you can run an apply, which implicitly does a plan and shows you the intended changes
+at the 'yes' prompt.
+
+```bash
+terraform apply
+```
+
+```bash
+yes
+```
+
+## Cleanup
+
+Run the following to remove the resources Terraform provisioned:
+
+```bash
+terraform destroy
+```
+```bash
+yes
+```