Fix permadiff on dataflow_flex_template_job (GoogleCloudPlatform#14038)

Author: FilipKubawskiOkta

mmv1/third_party/terraform/services/dataflow/resource_dataflow_flex_template_job.go.tmpl

@@ -477,12 +477,24 @@ func resourceDataflowFlexTemplateJobRead(d *schema.ResourceData, meta interface{
 	}
 	optionsMap := sdkPipelineOptions["options"].(map[string]interface{})
 
+	// sdkPipelineOptions is not always populated with these values, hence the fallback
+	if _, ok := d.GetOk("num_workers"); !ok && optionsMap["numWorkers"] == nil || optionsMap["numWorkers"] == 0 {
+		optionsMap["numWorkers"] = job.Environment.WorkerPools[0].NumWorkers
+	}
+	if _, ok := d.GetOk("max_num_workers"); !ok && optionsMap["maxNumWorkers"] == nil || optionsMap["maxNumWorkers"] == 0 {
+		optionsMap["maxNumWorkers"] = job.Environment.WorkerPools[0].AutoscalingSettings.MaxNumWorkers
+	}
+	if _, ok := d.GetOk("machine_type"); !ok && optionsMap["workerMachineType"] == nil || optionsMap["workerMachineType"] == "" {
+		optionsMap["workerMachineType"] = job.Environment.WorkerPools[0].MachineType
+	}
+	if _, ok := d.GetOk("sdk_container_image"); !ok && optionsMap["sdkContainerImage"] == nil || optionsMap["sdkContainerImage"] == "" {
+		optionsMap["sdkContainerImage"] = job.Environment.WorkerPools[0].WorkerHarnessContainerImage
+	}
+
+
 	if err := d.Set("temp_location", optionsMap["tempLocation"]); err != nil {
 		return fmt.Errorf("Error setting temp_gcs_location: %s", err)
 	}
-	if err := d.Set("network", optionsMap["network"]); err != nil {
-		return fmt.Errorf("Error setting network: %s", err)
-	}
 	if err := d.Set("num_workers", optionsMap["numWorkers"]); err != nil {
 		return fmt.Errorf("Error setting num_workers: %s", err)
 	}
@@ -495,10 +507,10 @@ func resourceDataflowFlexTemplateJobRead(d *schema.ResourceData, meta interface{
 	if err := d.Set("sdk_container_image", optionsMap["sdkContainerImage"]); err != nil {
 		return fmt.Errorf("Error setting sdk_container_image: %s", err)
 	}
-	if err := d.Set("network", optionsMap["network"]); err != nil {
+	if err := d.Set("network",  job.Environment.WorkerPools[0].Network); err != nil {
 		return fmt.Errorf("Error setting network: %s", err)
 	}
-	if err := d.Set("subnetwork", optionsMap["subnetwork"]); err != nil {
+	if err := d.Set("subnetwork", job.Environment.WorkerPools[0].Subnetwork); err != nil {
 		return fmt.Errorf("Error setting subnetwork: %s", err)
 	}
 	if err := d.Set("machine_type", optionsMap["workerMachineType"]); err != nil {

mmv1/third_party/terraform/services/dataflow/resource_dataflow_flex_template_job_test.go.tmpl

@@ -643,6 +643,50 @@ func TestAccDataflowFlexTemplateJob_enableStreamingEngine(t *testing.T) {
 	})
 }
 
+func TestAccDataflowFlexTemplateJob_workerPoolsFallback(t *testing.T) {
+	acctest.SkipIfVcr(t)
+	t.Parallel()
+
+	context1 := map[string]interface{}{
+		"random_id":    acctest.RandString(t, 10),
+		"max_workers":  2,
+		"num_workers":  1,
+		"machine_type": `"n1-standard-1"`,
+	}
+
+	context2 := map[string]interface{}{
+		"random_id":    context1["random_id"],
+		"max_workers":  3,
+		"num_workers":  2,
+		"machine_type": `"n1-standard-1"`,
+	}
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckDataflowJobDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataflowFlexTemplateJob_workerPoolFallback(context1),
+				Check: resource.ComposeTestCheckFunc(
+					testAccDataflowFlexJobExists(t, "google_dataflow_flex_template_job.flex_job", false),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "num_workers", "1"),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "max_workers", "2"),
+				),
+			},
+			{
+				Config: testAccDataflowFlexTemplateJob_workerPoolFallback(context2),
+				Check: resource.ComposeTestCheckFunc(
+					testAccDataflowFlexJobExists(t, "google_dataflow_flex_template_job.flex_job", true),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "num_workers", "2"),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "max_workers", "3"),
+				),
+			},
+		},
+	})
+}
+
+
 func testAccDataflowFlexTemplateJobHasNetwork(t *testing.T, res, expected string, wait bool) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		instanceTmpl, err := testAccDataflowFlexTemplateGetGeneratedInstanceTemplate(t, s, res)
@@ -2042,4 +2086,72 @@ resource "google_dataflow_flex_template_job" "flex_job" {
 }
 `, topicName, bucket, job)
 }
+
+func testAccDataflowFlexTemplateJob_workerPoolFallback(context map[string]interface{}) string {
+	return acctest.Nprintf(`
+resource "google_pubsub_topic" "example" {
+  name = "tf-test-topic-%{random_id}"
+}
+
+data "google_storage_bucket_object" "flex_template" {
+  name   = "latest/flex/Streaming_Data_Generator"
+  bucket = "dataflow-templates"
+}
+
+resource "google_storage_bucket" "bucket" {
+  name = "tf-test-bucket-%{random_id}"
+  location = "US-CENTRAL1"
+  force_destroy = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_object" "schema" {
+  name = "schema-%{random_id}.json"
+  bucket = google_storage_bucket.bucket.name
+  content = <<EOF
+{
+	"eventId": "{{"{{"}}uuid(){{"}}"}}",
+	"eventTimestamp": {{"{{"}}timestamp(){{"}}"}},
+	"ipv4": "{{"{{"}}ipv4(){{"}}"}}",
+	"ipv6": "{{"{{"}}ipv6(){{"}}"}}",
+	"country": "{{"{{"}}country(){{"}}"}}",
+	"username": "{{"{{"}}username(){{"}}"}}",
+	"quest": "{{"{{"}}random("A Break In the Ice", "Ghosts of Perdition", "Survive the Low Road"){{"}}"}}",
+	"score": {{"{{"}}integer(100, 10000){{"}}"}},
+	"completed": {{"{{"}}bool(){{"}}"}}
+}
+EOF
+}
+
+resource "google_compute_network" "test-network" {
+  name                    = "tf-test-network-%{random_id}"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "test-subnetwork" {
+  name          = "tf-test-subnetwork-%{random_id}"
+  ip_cidr_range = "192.168.0.0/24"
+  region		= "us-central1"
+  network       = google_compute_network.test-network.id
+}
+
+resource "google_dataflow_flex_template_job" "flex_job" {
+  name = "tf-test-flex-job-%{random_id}"
+  container_spec_gcs_path = "gs://${data.google_storage_bucket_object.flex_template.bucket}/${data.google_storage_bucket_object.flex_template.name}"
+  on_delete = "cancel"
+  parameters = {
+    schemaLocation = "gs://${google_storage_bucket_object.schema.bucket}/schema-%{random_id}.json"
+    qps = "1"
+    topic = google_pubsub_topic.example.id
+  }
+
+  max_workers = %{max_workers}
+  num_workers = %{num_workers}
+  machine_type = %{machine_type}
+  network = google_compute_network.test-network.name
+  subnetwork = google_compute_subnetwork.test-subnetwork.self_link
+}
+`, context)
+}
+
 {{- end }}