feat!: Update for TF and CI, add instance test (#59)

apeabody · web-flow · commit 77fa12c5da3f · 2022-07-29T12:01:38.000-07:00
feat: Update TPG for tenant_project, add test

* Update minimum TF to 0.13, TPG to 4.12, and TF modultes
* Update to CFT 1.5
* Convert template simple_example test to new instance test
* Update tenant_project output method
* Update instance example functionality
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
@@ -1,3 +1,17 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: "Close stale issues"
 on:
   schedule:
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
-DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.0
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.5
 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools
 REGISTRY_URL := gcr.io/cloud-foundation-cicd
 
diff --git a/README.md b/README.md
@@ -54,8 +54,8 @@ These sections describe requirements for using this module.
 
 The following dependencies must be available:
 
-- [Terraform][terraform] v0.12
-- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0
+- [Terraform][terraform] v0.13+
+- [Terraform Provider for GCP][terraform-provider-gcp] plugin v4.12
 
 ### Service Account
 
diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
@@ -38,4 +38,4 @@ tags:
 - 'integration'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5'
diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml
@@ -21,4 +21,4 @@ tags:
 - 'lint'
 substitutions:
   _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools'
-  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.0'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5'
diff --git a/examples/hub_artifact/versions.tf b/examples/hub_artifact/versions.tf
@@ -1,3 +1,19 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 terraform {
   required_providers {
     cdap = {
diff --git a/examples/instance/README.md b/examples/instance/README.md
@@ -7,11 +7,14 @@ This example illustrates how to use the `data-fusion` module.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| data\_fusion\_service\_account | Google Managed Data Fusion Service Account Email | `any` | n/a | yes |
 | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes |
 
 ## Outputs
 
-No output.
+| Name | Description |
+|------|-------------|
+| instance\_id | The created Google Data Fusion instance ID |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
diff --git a/examples/instance/main.tf b/examples/instance/main.tf
@@ -17,7 +17,9 @@
 module "data_fusion" {
   source = "../../"
 
-  name    = "example-instance"
-  project = var.project_id
-  region  = "us-central1"
+  name                        = "example-instance"
+  project                     = var.project_id
+  region                      = "us-central1"
+  network                     = "default"
+  data_fusion_service_account = var.data_fusion_service_account
 }
diff --git a/examples/instance/outputs.tf b/examples/instance/outputs.tf
@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Google LLC
+ * Copyright 2022 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
-terraform {
-  required_version = ">= 0.12"
+output "instance_id" {
+  description = "The created Google Data Fusion instance ID"
+  value       = module.data_fusion.instance.id
 }
diff --git a/examples/instance/simple/README.md b/examples/instance/simple/README.md
@@ -7,6 +7,7 @@ This example illustrates how to use the `data-fusion` module.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| data\_fusion\_service\_account | Google Managed Data Fusion Service Account Email | `any` | n/a | yes |
 | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes |
 
 ## Outputs
diff --git a/examples/instance/simple/main.tf b/examples/instance/simple/main.tf
@@ -15,9 +15,11 @@
  */
 
 module "data_fusion" {
-  source = "../../"
+  source = "../../../"
 
-  name    = "example-instance"
-  project = var.project_id
-  region  = "us-central1"
+  name                        = "example-instance"
+  project                     = var.project_id
+  region                      = "us-central1"
+  network                     = "default"
+  data_fusion_service_account = var.data_fusion_service_account
 }
diff --git a/examples/instance/simple/variables.tf b/examples/instance/simple/variables.tf
@@ -18,3 +18,7 @@ variable "project_id" {
   description = "The ID of the project in which to provision resources."
   type        = string
 }
+
+variable "data_fusion_service_account" {
+  description = "Google Managed Data Fusion Service Account Email"
+}
diff --git a/examples/instance/variables.tf b/examples/instance/variables.tf
@@ -18,3 +18,7 @@ variable "project_id" {
   description = "The ID of the project in which to provision resources."
   type        = string
 }
+
+variable "data_fusion_service_account" {
+  description = "Google Managed Data Fusion Service Account Email"
+}
diff --git a/examples/instance/versions.tf b/examples/instance/versions.tf
diff --git a/kitchen.yml b/kitchen.yml
@@ -15,6 +15,8 @@
 ---
 driver:
   name: terraform
+  command_timeout: 2700
+  verify_version: false
 
 provisioner:
   name: terraform
@@ -26,18 +28,13 @@ platforms:
   - name: default
 
 suites:
-  - name: simple_example
+  - name: instance
     driver:
-      root_module_directory: test/fixtures/simple_example/
+      root_module_directory: test/fixtures/instance/
     verifier:
       color: false
       systems:
-        - name: simple_example local
+        - name: instance local
           backend: local
           controls:
             - gcloud
-            - gsutil
-        - name: simple_example gcp
-          backend: gcp
-          controls:
-            - gcp
diff --git a/main.tf b/main.tf
@@ -29,14 +29,14 @@ module "data_fusion_network" {
 module "instance" {
   source = "./modules/instance"
 
-  name        = var.name
-  project     = var.project
-  description = var.description
-  region      = var.region
-  type        = var.type
-  labels      = var.labels
-  datafusion_version   = var.datafusion_version
-  options     = var.options
+  name               = var.name
+  project            = var.project
+  description        = var.description
+  region             = var.region
+  type               = var.type
+  labels             = var.labels
+  datafusion_version = var.datafusion_version
+  options            = var.options
   network_config = {
     network       = module.data_fusion_network.data_fusion_vpc.network_name
     ip_allocation = module.data_fusion_network.data_fusion_ip_allocation
diff --git a/modules/dataproc_profile/versions.tf b/modules/dataproc_profile/versions.tf
@@ -22,6 +22,10 @@ terraform {
       source  = "hashicorp/google"
       version = ">= 3.53, < 5.0"
     }
+    cdap = {
+      source  = "GoogleCloudPlatform/cdap"
+      version = "~> 0.9.0"
+    }
   }
 
   provider_meta "google" {
diff --git a/modules/hub_artifact/versions.tf b/modules/hub_artifact/versions.tf
@@ -22,6 +22,10 @@ terraform {
       source  = "hashicorp/google"
       version = ">= 3.53, < 5.0"
     }
+    cdap = {
+      source  = "GoogleCloudPlatform/cdap"
+      version = "~> 0.9.0"
+    }
   }
 
   provider_meta "google" {
diff --git a/modules/instance/outputs.tf b/modules/instance/outputs.tf
@@ -19,13 +19,9 @@ output "instance" {
   description = "The created CDF instance"
 }
 
-locals {
-  tenant_project_re = "cloud-datafusion-management-sa@([\\w-]+).iam.gserviceaccount.com"
-}
-
 output "tenant_project" {
-  value       = regex(local.tenant_project_re, google_data_fusion_instance.instance.service_account)[0]
   description = "The Google managed tenant project ID in which the instance will run its jobs"
+  value       = google_data_fusion_instance.instance.tenant_project_id
 }
 
 output "service_account" {
diff --git a/modules/instance/versions.tf b/modules/instance/versions.tf
@@ -15,5 +15,13 @@
  */
 
 terraform {
-  required_version = "~> 0.12.6"
+  required_version = ">= 0.13"
+
+  provider_meta "google" {
+    module_name = "blueprints/terraform/terraform-google-data-fusion:instance/v1.1.0"
+  }
+
+  provider_meta "google-beta" {
+    module_name = "blueprints/terraform/terraform-google-data-fusion:instance/v1.1.0"
+  }
 }
diff --git a/modules/namespace/versions.tf b/modules/namespace/versions.tf
@@ -22,6 +22,11 @@ terraform {
       source  = "hashicorp/google"
       version = ">= 3.53, < 5.0"
     }
+    cdap = {
+      source  = "GoogleCloudPlatform/cdap"
+      version = "~> 0.9.0"
+    }
+
   }
 
   provider_meta "google" {
diff --git a/modules/private_network/README.md b/modules/private_network/README.md
@@ -1,6 +1,6 @@
 # Data Fusion Private Network Setup
 
-This module can be used to deploy the networking pre-reqs for a 
+This module can be used to deploy the networking pre-reqs for a
 [Private IP Data Fusion Instance](https://cloud.google.com/data-fusion/docs/how-to/create-private-ip).
 Namely:
 - VPC Network
diff --git a/modules/private_network/main.tf b/modules/private_network/main.tf
@@ -16,7 +16,7 @@
 
 module "vpc" {
   source       = "terraform-google-modules/network/google"
-  version      = "~> 2.2"
+  version      = "~> 5.1"
   project_id   = var.project_id
   network_name = var.network_name
   routing_mode = "GLOBAL"
@@ -31,13 +31,12 @@ module "vpc" {
   ]
 }
 
-module "peering" {
-  source                     = "terraform-google-modules/network/google//modules/network-peering"
-  prefix                     = "data-fusion-peering"
-  local_network              = module.vpc.network_self_link
-  peer_network               = "projects/${var.tenant_project}/global/networks/${var.region}-${var.instance}"
-  export_local_custom_routes = true
-  export_peer_custom_routes  = true
+resource "google_compute_network_peering" "data-fusion-peering" {
+  name                 = "data-fusion-peering"
+  network              = module.vpc.network_self_link
+  peer_network         = "projects/${var.tenant_project}/global/networks/${var.region}-${var.instance}"
+  export_custom_routes = true
+  import_custom_routes = true
 }
 
 resource "google_compute_global_address" "data_fusion_private_ip_alloc" {
diff --git a/modules/private_network/variables.tf b/modules/private_network/variables.tf
@@ -34,6 +34,7 @@ variable "tenant_project" {
   description = "Private Data Fusion instance ID"
   type        = string
 }
+
 variable "data_fusion_service_account" {
   description = "The Google managed Data Fusion Service account"
   type        = string
diff --git a/modules/private_network/versions.tf b/modules/private_network/versions.tf
@@ -20,7 +20,7 @@ terraform {
 
     google = {
       source  = "hashicorp/google"
-      version = ">= 3.53, < 5.0"
+      version = ">= 4.12, < 5.0"
     }
   }
 
diff --git a/modules/wait_healthy/README.md b/modules/wait_healthy/README.md
@@ -73,8 +73,8 @@ These sections describe requirements for using this module.
 
 The following dependencies must be available:
 
-- [Terraform][terraform] v0.12
-- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0
+- [Terraform][terraform] v0.13+
+- [Terraform Provider for GCP][terraform-provider-gcp] plugin v3.53+
 
 ### Service Account
 
diff --git a/test/fixtures/instance/main.tf b/test/fixtures/instance/main.tf
@@ -14,12 +14,9 @@
  * limitations under the License.
  */
 
-terraform {
-  required_version = ">= 0.13"
-  required_providers {
-    random = {
-      source  = "hashicorp/random"
-      version = "~> 2.0"
-    }
-  }
+module "example" {
+  source = "../../../examples/instance"
+
+  project_id                  = var.project_id
+  data_fusion_service_account = var.sa_key
 }
diff --git a/test/fixtures/instance/outputs.tf b/test/fixtures/instance/outputs.tf
@@ -14,12 +14,12 @@
  * limitations under the License.
  */
 
-output "bucket_name" {
-  description = "The name of the bucket."
-  value       = module.example.bucket_name
-}
-
 output "project_id" {
   description = "The ID of the project in which resources are provisioned."
   value       = var.project_id
 }
+
+output "instance_id" {
+  description = "The created Google Data Fusion instance id"
+  value       = module.example.instance_id
+}
diff --git a/test/fixtures/instance/variables.tf b/test/fixtures/instance/variables.tf
diff --git a/test/fixtures/simple_example/main.tf b/test/fixtures/simple_example/main.tf
diff --git a/test/integration/instance/controls/gcloud.rb b/test/integration/instance/controls/gcloud.rb
diff --git a/test/integration/instance/inspec.yml b/test/integration/instance/inspec.yml
diff --git a/test/integration/simple_example/controls/gcp.rb b/test/integration/simple_example/controls/gcp.rb
diff --git a/test/integration/simple_example/controls/gsutil.rb b/test/integration/simple_example/controls/gsutil.rb
diff --git a/test/setup/main.tf b/test/setup/main.tf
diff --git a/test/setup/versions.tf b/test/setup/versions.tf
