Fix permadiff on dataflow_flex_template_job (#14038) (#10168)

[upstream:ab863f2bdc7a243aeb78350e335959f8d4cfb84e]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/14038.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+dataflow: Fields `network`, `subnetwork`, `numWorkers`, `maxNumWorkers` and `machineType` will no longer cause permadiff on `dataflow_flex_template_job`
+```

google-beta/services/dataflow/resource_dataflow_flex_template_job.go

@@ -492,12 +492,23 @@ func resourceDataflowFlexTemplateJobRead(d *schema.ResourceData, meta interface{
 	}
 	optionsMap := sdkPipelineOptions["options"].(map[string]interface{})
 
+	// sdkPipelineOptions is not always populated with these values, hence the fallback
+	if _, ok := d.GetOk("num_workers"); !ok && optionsMap["numWorkers"] == nil || optionsMap["numWorkers"] == 0 {
+		optionsMap["numWorkers"] = job.Environment.WorkerPools[0].NumWorkers
+	}
+	if _, ok := d.GetOk("max_num_workers"); !ok && optionsMap["maxNumWorkers"] == nil || optionsMap["maxNumWorkers"] == 0 {
+		optionsMap["maxNumWorkers"] = job.Environment.WorkerPools[0].AutoscalingSettings.MaxNumWorkers
+	}
+	if _, ok := d.GetOk("machine_type"); !ok && optionsMap["workerMachineType"] == nil || optionsMap["workerMachineType"] == "" {
+		optionsMap["workerMachineType"] = job.Environment.WorkerPools[0].MachineType
+	}
+	if _, ok := d.GetOk("sdk_container_image"); !ok && optionsMap["sdkContainerImage"] == nil || optionsMap["sdkContainerImage"] == "" {
+		optionsMap["sdkContainerImage"] = job.Environment.WorkerPools[0].WorkerHarnessContainerImage
+	}
+
 	if err := d.Set("temp_location", optionsMap["tempLocation"]); err != nil {
 		return fmt.Errorf("Error setting temp_gcs_location: %s", err)
 	}
-	if err := d.Set("network", optionsMap["network"]); err != nil {
-		return fmt.Errorf("Error setting network: %s", err)
-	}
 	if err := d.Set("num_workers", optionsMap["numWorkers"]); err != nil {
 		return fmt.Errorf("Error setting num_workers: %s", err)
 	}
@@ -510,10 +521,10 @@ func resourceDataflowFlexTemplateJobRead(d *schema.ResourceData, meta interface{
 	if err := d.Set("sdk_container_image", optionsMap["sdkContainerImage"]); err != nil {
 		return fmt.Errorf("Error setting sdk_container_image: %s", err)
 	}
-	if err := d.Set("network", optionsMap["network"]); err != nil {
+	if err := d.Set("network", job.Environment.WorkerPools[0].Network); err != nil {
 		return fmt.Errorf("Error setting network: %s", err)
 	}
-	if err := d.Set("subnetwork", optionsMap["subnetwork"]); err != nil {
+	if err := d.Set("subnetwork", job.Environment.WorkerPools[0].Subnetwork); err != nil {
 		return fmt.Errorf("Error setting subnetwork: %s", err)
 	}
 	if err := d.Set("machine_type", optionsMap["workerMachineType"]); err != nil {

google-beta/services/dataflow/resource_dataflow_flex_template_job_test.go

@@ -658,6 +658,49 @@ func TestAccDataflowFlexTemplateJob_enableStreamingEngine(t *testing.T) {
 	})
 }
 
+func TestAccDataflowFlexTemplateJob_workerPoolsFallback(t *testing.T) {
+	acctest.SkipIfVcr(t)
+	t.Parallel()
+
+	context1 := map[string]interface{}{
+		"random_id":    acctest.RandString(t, 10),
+		"max_workers":  2,
+		"num_workers":  1,
+		"machine_type": `"n1-standard-1"`,
+	}
+
+	context2 := map[string]interface{}{
+		"random_id":    context1["random_id"],
+		"max_workers":  3,
+		"num_workers":  2,
+		"machine_type": `"n1-standard-1"`,
+	}
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckDataflowJobDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataflowFlexTemplateJob_workerPoolFallback(context1),
+				Check: resource.ComposeTestCheckFunc(
+					testAccDataflowFlexJobExists(t, "google_dataflow_flex_template_job.flex_job", false),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "num_workers", "1"),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "max_workers", "2"),
+				),
+			},
+			{
+				Config: testAccDataflowFlexTemplateJob_workerPoolFallback(context2),
+				Check: resource.ComposeTestCheckFunc(
+					testAccDataflowFlexJobExists(t, "google_dataflow_flex_template_job.flex_job", true),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "num_workers", "2"),
+					resource.TestCheckResourceAttr("google_dataflow_flex_template_job.flex_job", "max_workers", "3"),
+				),
+			},
+		},
+	})
+}
+
 func testAccDataflowFlexTemplateJobHasNetwork(t *testing.T, res, expected string, wait bool) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		instanceTmpl, err := testAccDataflowFlexTemplateGetGeneratedInstanceTemplate(t, s, res)
@@ -2057,3 +2100,70 @@ resource "google_dataflow_flex_template_job" "flex_job" {
 }
 `, topicName, bucket, job)
 }
+
+func testAccDataflowFlexTemplateJob_workerPoolFallback(context map[string]interface{}) string {
+	return acctest.Nprintf(`
+resource "google_pubsub_topic" "example" {
+  name = "tf-test-topic-%{random_id}"
+}
+
+data "google_storage_bucket_object" "flex_template" {
+  name   = "latest/flex/Streaming_Data_Generator"
+  bucket = "dataflow-templates"
+}
+
+resource "google_storage_bucket" "bucket" {
+  name = "tf-test-bucket-%{random_id}"
+  location = "US-CENTRAL1"
+  force_destroy = true
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_object" "schema" {
+  name = "schema-%{random_id}.json"
+  bucket = google_storage_bucket.bucket.name
+  content = <<EOF
+{
+	"eventId": "{{uuid()}}",
+	"eventTimestamp": {{timestamp()}},
+	"ipv4": "{{ipv4()}}",
+	"ipv6": "{{ipv6()}}",
+	"country": "{{country()}}",
+	"username": "{{username()}}",
+	"quest": "{{random("A Break In the Ice", "Ghosts of Perdition", "Survive the Low Road")}}",
+	"score": {{integer(100, 10000)}},
+	"completed": {{bool()}}
+}
+EOF
+}
+
+resource "google_compute_network" "test-network" {
+  name                    = "tf-test-network-%{random_id}"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "test-subnetwork" {
+  name          = "tf-test-subnetwork-%{random_id}"
+  ip_cidr_range = "192.168.0.0/24"
+  region		= "us-central1"
+  network       = google_compute_network.test-network.id
+}
+
+resource "google_dataflow_flex_template_job" "flex_job" {
+  name = "tf-test-flex-job-%{random_id}"
+  container_spec_gcs_path = "gs://${data.google_storage_bucket_object.flex_template.bucket}/${data.google_storage_bucket_object.flex_template.name}"
+  on_delete = "cancel"
+  parameters = {
+    schemaLocation = "gs://${google_storage_bucket_object.schema.bucket}/schema-%{random_id}.json"
+    qps = "1"
+    topic = google_pubsub_topic.example.id
+  }
+
+  max_workers = %{max_workers}
+  num_workers = %{num_workers}
+  machine_type = %{machine_type}
+  network = google_compute_network.test-network.name
+  subnetwork = google_compute_subnetwork.test-subnetwork.self_link
+}
+`, context)
+}