diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fcf18fe..e44714e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,3 +1,17 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + name: "Close stale issues" on: schedule: diff --git a/Makefile b/Makefile index 5b71d3c..975c6b1 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.0 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.5 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/README.md b/README.md index 7a799b2..21e1e05 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ These sections describe requirements for using this module. The following dependencies must be available: -- [Terraform][terraform] v0.12 -- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0 +- [Terraform][terraform] v0.13+ +- [Terraform Provider for GCP][terraform-provider-gcp] plugin v4.12 ### Service Account diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 2fe9a02..62e8d94 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -38,4 +38,4 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index d570491..2a0c65e 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -21,4 +21,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.5' diff --git a/examples/hub_artifact/versions.tf b/examples/hub_artifact/versions.tf index 15bd29b..b181526 100644 --- a/examples/hub_artifact/versions.tf +++ b/examples/hub_artifact/versions.tf @@ -1,3 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + terraform { required_providers { cdap = { diff --git a/examples/instance/README.md b/examples/instance/README.md index 5931695..a0761d6 100644 --- a/examples/instance/README.md +++ b/examples/instance/README.md @@ -7,11 +7,14 @@ This example illustrates how to use the `data-fusion` module. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| data\_fusion\_service\_account | Google Managed Data Fusion Service Account Email | `any` | n/a | yes | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | ## Outputs -No output. +| Name | Description | +|------|-------------| +| instance\_id | The created Google Data Fusion instance ID | diff --git a/examples/instance/main.tf b/examples/instance/main.tf index 54193d9..f36b75f 100644 --- a/examples/instance/main.tf +++ b/examples/instance/main.tf @@ -17,7 +17,9 @@ module "data_fusion" { source = "../../" - name = "example-instance" - project = var.project_id - region = "us-central1" + name = "example-instance" + project = var.project_id + region = "us-central1" + network = "default" + data_fusion_service_account = var.data_fusion_service_account } diff --git a/examples/instance/simple/versions.tf b/examples/instance/outputs.tf similarity index 78% rename from examples/instance/simple/versions.tf rename to examples/instance/outputs.tf index 0bad581..fabde5e 100644 --- a/examples/instance/simple/versions.tf +++ b/examples/instance/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2020 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ -terraform { - required_version = ">= 0.12" +output "instance_id" { + description = "The created Google Data Fusion instance ID" + value = module.data_fusion.instance.id } diff --git a/examples/instance/simple/README.md b/examples/instance/simple/README.md index 5931695..1df2a21 100644 --- a/examples/instance/simple/README.md +++ b/examples/instance/simple/README.md @@ -7,6 +7,7 @@ This example illustrates how to use the `data-fusion` module. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| data\_fusion\_service\_account | Google Managed Data Fusion Service Account Email | `any` | n/a | yes | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | ## Outputs diff --git a/examples/instance/simple/main.tf b/examples/instance/simple/main.tf index 54193d9..a8d5a43 100644 --- a/examples/instance/simple/main.tf +++ b/examples/instance/simple/main.tf @@ -15,9 +15,11 @@ */ module "data_fusion" { - source = "../../" + source = "../../../" - name = "example-instance" - project = var.project_id - region = "us-central1" + name = "example-instance" + project = var.project_id + region = "us-central1" + network = "default" + data_fusion_service_account = var.data_fusion_service_account } diff --git a/examples/instance/simple/variables.tf b/examples/instance/simple/variables.tf index bba3500..39797c9 100644 --- a/examples/instance/simple/variables.tf +++ b/examples/instance/simple/variables.tf @@ -18,3 +18,7 @@ variable "project_id" { description = "The ID of the project in which to provision resources." type = string } + +variable "data_fusion_service_account" { + description = "Google Managed Data Fusion Service Account Email" +} diff --git a/examples/instance/variables.tf b/examples/instance/variables.tf index bba3500..39797c9 100644 --- a/examples/instance/variables.tf +++ b/examples/instance/variables.tf @@ -18,3 +18,7 @@ variable "project_id" { description = "The ID of the project in which to provision resources." type = string } + +variable "data_fusion_service_account" { + description = "Google Managed Data Fusion Service Account Email" +} diff --git a/examples/instance/versions.tf b/examples/instance/versions.tf deleted file mode 100644 index 0bad581..0000000 --- a/examples/instance/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 0.12" -} diff --git a/kitchen.yml b/kitchen.yml index f3c9c35..3aa9240 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -15,6 +15,8 @@ --- driver: name: terraform + command_timeout: 2700 + verify_version: false provisioner: name: terraform @@ -26,18 +28,13 @@ platforms: - name: default suites: - - name: simple_example + - name: instance driver: - root_module_directory: test/fixtures/simple_example/ + root_module_directory: test/fixtures/instance/ verifier: color: false systems: - - name: simple_example local + - name: instance local backend: local controls: - gcloud - - gsutil - - name: simple_example gcp - backend: gcp - controls: - - gcp diff --git a/main.tf b/main.tf index bf11e80..1d60728 100644 --- a/main.tf +++ b/main.tf @@ -29,14 +29,14 @@ module "data_fusion_network" { module "instance" { source = "./modules/instance" - name = var.name - project = var.project - description = var.description - region = var.region - type = var.type - labels = var.labels - datafusion_version = var.datafusion_version - options = var.options + name = var.name + project = var.project + description = var.description + region = var.region + type = var.type + labels = var.labels + datafusion_version = var.datafusion_version + options = var.options network_config = { network = module.data_fusion_network.data_fusion_vpc.network_name ip_allocation = module.data_fusion_network.data_fusion_ip_allocation diff --git a/modules/dataproc_profile/versions.tf b/modules/dataproc_profile/versions.tf index aaa59d8..15ce57d 100644 --- a/modules/dataproc_profile/versions.tf +++ b/modules/dataproc_profile/versions.tf @@ -22,6 +22,10 @@ terraform { source = "hashicorp/google" version = ">= 3.53, < 5.0" } + cdap = { + source = "GoogleCloudPlatform/cdap" + version = "~> 0.9.0" + } } provider_meta "google" { diff --git a/modules/hub_artifact/versions.tf b/modules/hub_artifact/versions.tf index f52c545..1910dd4 100644 --- a/modules/hub_artifact/versions.tf +++ b/modules/hub_artifact/versions.tf @@ -22,6 +22,10 @@ terraform { source = "hashicorp/google" version = ">= 3.53, < 5.0" } + cdap = { + source = "GoogleCloudPlatform/cdap" + version = "~> 0.9.0" + } } provider_meta "google" { diff --git a/modules/instance/outputs.tf b/modules/instance/outputs.tf index 21c9398..a43242e 100644 --- a/modules/instance/outputs.tf +++ b/modules/instance/outputs.tf @@ -19,13 +19,9 @@ output "instance" { description = "The created CDF instance" } -locals { - tenant_project_re = "cloud-datafusion-management-sa@([\\w-]+).iam.gserviceaccount.com" -} - output "tenant_project" { - value = regex(local.tenant_project_re, google_data_fusion_instance.instance.service_account)[0] description = "The Google managed tenant project ID in which the instance will run its jobs" + value = google_data_fusion_instance.instance.tenant_project_id } output "service_account" { diff --git a/modules/instance/versions.tf b/modules/instance/versions.tf index 3164c25..a8b68bc 100644 --- a/modules/instance/versions.tf +++ b/modules/instance/versions.tf @@ -15,5 +15,13 @@ */ terraform { - required_version = "~> 0.12.6" + required_version = ">= 0.13" + + provider_meta "google" { + module_name = "blueprints/terraform/terraform-google-data-fusion:instance/v1.1.0" + } + + provider_meta "google-beta" { + module_name = "blueprints/terraform/terraform-google-data-fusion:instance/v1.1.0" + } } diff --git a/modules/namespace/versions.tf b/modules/namespace/versions.tf index bd3f3d4..0ed99be 100644 --- a/modules/namespace/versions.tf +++ b/modules/namespace/versions.tf @@ -22,6 +22,11 @@ terraform { source = "hashicorp/google" version = ">= 3.53, < 5.0" } + cdap = { + source = "GoogleCloudPlatform/cdap" + version = "~> 0.9.0" + } + } provider_meta "google" { diff --git a/modules/private_network/README.md b/modules/private_network/README.md index bff6366..eddf52d 100644 --- a/modules/private_network/README.md +++ b/modules/private_network/README.md @@ -1,6 +1,6 @@ # Data Fusion Private Network Setup -This module can be used to deploy the networking pre-reqs for a +This module can be used to deploy the networking pre-reqs for a [Private IP Data Fusion Instance](https://cloud.google.com/data-fusion/docs/how-to/create-private-ip). Namely: - VPC Network diff --git a/modules/private_network/main.tf b/modules/private_network/main.tf index 0b2adec..d73266e 100644 --- a/modules/private_network/main.tf +++ b/modules/private_network/main.tf @@ -16,7 +16,7 @@ module "vpc" { source = "terraform-google-modules/network/google" - version = "~> 2.2" + version = "~> 5.1" project_id = var.project_id network_name = var.network_name routing_mode = "GLOBAL" @@ -31,13 +31,12 @@ module "vpc" { ] } -module "peering" { - source = "terraform-google-modules/network/google//modules/network-peering" - prefix = "data-fusion-peering" - local_network = module.vpc.network_self_link - peer_network = "projects/${var.tenant_project}/global/networks/${var.region}-${var.instance}" - export_local_custom_routes = true - export_peer_custom_routes = true +resource "google_compute_network_peering" "data-fusion-peering" { + name = "data-fusion-peering" + network = module.vpc.network_self_link + peer_network = "projects/${var.tenant_project}/global/networks/${var.region}-${var.instance}" + export_custom_routes = true + import_custom_routes = true } resource "google_compute_global_address" "data_fusion_private_ip_alloc" { diff --git a/modules/private_network/variables.tf b/modules/private_network/variables.tf index 6a2a427..3aa0ec2 100644 --- a/modules/private_network/variables.tf +++ b/modules/private_network/variables.tf @@ -34,6 +34,7 @@ variable "tenant_project" { description = "Private Data Fusion instance ID" type = string } + variable "data_fusion_service_account" { description = "The Google managed Data Fusion Service account" type = string diff --git a/modules/private_network/versions.tf b/modules/private_network/versions.tf index 9ef5464..d69159e 100644 --- a/modules/private_network/versions.tf +++ b/modules/private_network/versions.tf @@ -20,7 +20,7 @@ terraform { google = { source = "hashicorp/google" - version = ">= 3.53, < 5.0" + version = ">= 4.12, < 5.0" } } diff --git a/modules/wait_healthy/README.md b/modules/wait_healthy/README.md index 9e0a7d7..cb34575 100644 --- a/modules/wait_healthy/README.md +++ b/modules/wait_healthy/README.md @@ -73,8 +73,8 @@ These sections describe requirements for using this module. The following dependencies must be available: -- [Terraform][terraform] v0.12 -- [Terraform Provider for GCP][terraform-provider-gcp] plugin v2.0 +- [Terraform][terraform] v0.13+ +- [Terraform Provider for GCP][terraform-provider-gcp] plugin v3.53+ ### Service Account diff --git a/test/fixtures/simple_example/versions.tf b/test/fixtures/instance/main.tf similarity index 79% rename from test/fixtures/simple_example/versions.tf rename to test/fixtures/instance/main.tf index b69cac7..c0261a1 100644 --- a/test/fixtures/simple_example/versions.tf +++ b/test/fixtures/instance/main.tf @@ -14,12 +14,9 @@ * limitations under the License. */ -terraform { - required_version = ">= 0.13" - required_providers { - random = { - source = "hashicorp/random" - version = "~> 2.0" - } - } +module "example" { + source = "../../../examples/instance" + + project_id = var.project_id + data_fusion_service_account = var.sa_key } diff --git a/test/fixtures/simple_example/outputs.tf b/test/fixtures/instance/outputs.tf similarity index 85% rename from test/fixtures/simple_example/outputs.tf rename to test/fixtures/instance/outputs.tf index 2e41354..3dce547 100644 --- a/test/fixtures/simple_example/outputs.tf +++ b/test/fixtures/instance/outputs.tf @@ -14,12 +14,12 @@ * limitations under the License. */ -output "bucket_name" { - description = "The name of the bucket." - value = module.example.bucket_name -} - output "project_id" { description = "The ID of the project in which resources are provisioned." value = var.project_id } + +output "instance_id" { + description = "The created Google Data Fusion instance id" + value = module.example.instance_id +} diff --git a/test/fixtures/simple_example/variables.tf b/test/fixtures/instance/variables.tf similarity index 89% rename from test/fixtures/simple_example/variables.tf rename to test/fixtures/instance/variables.tf index c1a5c77..f5d0f81 100644 --- a/test/fixtures/simple_example/variables.tf +++ b/test/fixtures/instance/variables.tf @@ -18,3 +18,8 @@ variable "project_id" { description = "The ID of the project in which to provision resources." type = string } + +variable "sa_key" { + description = "Service Account key" + type = string +} diff --git a/test/fixtures/simple_example/main.tf b/test/fixtures/simple_example/main.tf deleted file mode 100644 index 592bc8a..0000000 --- a/test/fixtures/simple_example/main.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_pet" "main" { - length = 1 - prefix = "simple-example" - separator = "-" -} - -module "example" { - source = "../../../examples/simple_example" - - project_id = var.project_id - bucket_name = random_pet.main.id -} diff --git a/test/integration/simple_example/controls/gcloud.rb b/test/integration/instance/controls/gcloud.rb similarity index 60% rename from test/integration/simple_example/controls/gcloud.rb rename to test/integration/instance/controls/gcloud.rb index d2a2609..0ba7007 100644 --- a/test/integration/simple_example/controls/gcloud.rb +++ b/test/integration/instance/controls/gcloud.rb @@ -13,11 +13,24 @@ # limitations under the License. control "gcloud" do - title "gcloud" + title "Google Data Fusion instance" - describe command("gcloud --project=#{attribute("project_id")} services list --enabled") do + describe command("gcloud beta data-fusion instances describe #{attribute("instance_id")} --format=json") do its(:exit_status) { should eq 0 } - its(:stderr) { should eq "" } - its(:stdout) { should match "storage-api.googleapis.com" } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "instance" do + it "is running" do + expect(data['state']).to eq('RUNNING') + end + end end end diff --git a/test/integration/simple_example/inspec.yml b/test/integration/instance/inspec.yml similarity index 82% rename from test/integration/simple_example/inspec.yml rename to test/integration/instance/inspec.yml index ae2da09..6590f6a 100644 --- a/test/integration/simple_example/inspec.yml +++ b/test/integration/instance/inspec.yml @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: simple_example -depends: - - name: inspec-gcp - git: https://github.com/inspec/inspec-gcp.git - tag: v0.10.0 +name: instance attributes: - name: project_id required: true type: string - - name: bucket_name +attributes: + - name: instance_id required: true type: string diff --git a/test/integration/simple_example/controls/gcp.rb b/test/integration/simple_example/controls/gcp.rb deleted file mode 100644 index 3b5382b..0000000 --- a/test/integration/simple_example/controls/gcp.rb +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -control "gcp" do - title "GCP Resources" - - describe google_storage_bucket(name: attribute("bucket_name")) do - it { should exist } - end -end diff --git a/test/integration/simple_example/controls/gsutil.rb b/test/integration/simple_example/controls/gsutil.rb deleted file mode 100644 index 692309f..0000000 --- a/test/integration/simple_example/controls/gsutil.rb +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -control "gsutil" do - title "gsutil" - - describe command("gsutil ls -p #{attribute("project_id")}") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq "" } - its(:stdout) { should match "gs://#{attribute("bucket_name")}" } - end -end diff --git a/test/setup/main.tf b/test/setup/main.tf index 5d68304..cb98940 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -16,9 +16,9 @@ module "project" { source = "terraform-google-modules/project-factory/google" - version = "~> 10.0" + version = "~> 13.0" - name = "ci-data_fusion" + name = "ci-data-fusion" random_project_id = "true" org_id = var.org_id folder_id = var.folder_id @@ -27,6 +27,7 @@ module "project" { activate_apis = [ "cloudresourcemanager.googleapis.com", "storage-api.googleapis.com", - "serviceusage.googleapis.com" + "serviceusage.googleapis.com", + "datafusion.googleapis.com" ] } diff --git a/test/setup/versions.tf b/test/setup/versions.tf index 1bdf898..3da4fc4 100644 --- a/test/setup/versions.tf +++ b/test/setup/versions.tf @@ -19,11 +19,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "~> 3.53.0" + version = ">= 3.53.0, < 5.0" } google-beta = { source = "hashicorp/google-beta" - version = "~> 3.53.0" + version = ">= 3.53.0, < 5.0" } } }