Update logging sinks to tf1.3 in resman modules

fast/stages/00-bootstrap/organization.tf

@@ -192,13 +192,9 @@ module "organization" {
   }
   logging_sinks = {
     for name, attrs in var.log_sinks : name => {
-      bq_partitioned_table = attrs.type == "bigquery"
-      destination          = local.log_sink_destinations[name].id
-      exclusions           = {}
-      filter               = attrs.filter
-      iam                  = true
-      include_children     = true
-      type                 = attrs.type
+      destination                    = local.log_sink_destinations[name].as_logging_destination
+      filter                         = attrs.filter
+      bigquery_use_partitioned_table = attrs.type == "bigquery"
     }
   }
 }

modules/bigquery-dataset/README.md

@@ -196,13 +196,14 @@ module "bigquery-dataset" {
 
 | name | description | sensitive |
 |---|---|:---:|
-| [dataset](outputs.tf#L17) | Dataset resource. |  |
-| [dataset_id](outputs.tf#L22) | Dataset id. |  |
-| [id](outputs.tf#L34) | Fully qualified dataset id. |  |
-| [self_link](outputs.tf#L46) | Dataset self link. |  |
-| [table_ids](outputs.tf#L58) | Map of fully qualified table ids keyed by table ids. |  |
-| [tables](outputs.tf#L63) | Table resources. |  |
-| [view_ids](outputs.tf#L68) | Map of fully qualified view ids keyed by view ids. |  |
-| [views](outputs.tf#L73) | View resources. |  |
+| [as_logging_destination](outputs.tf#L17) | Parameters to use this dataset as a log sink destination. |  |
+| [dataset](outputs.tf#L32) | Dataset resource. |  |
+| [dataset_id](outputs.tf#L37) | Dataset id. |  |
+| [id](outputs.tf#L49) | Fully qualified dataset id. |  |
+| [self_link](outputs.tf#L61) | Dataset self link. |  |
+| [table_ids](outputs.tf#L73) | Map of fully qualified table ids keyed by table ids. |  |
+| [tables](outputs.tf#L78) | Table resources. |  |
+| [view_ids](outputs.tf#L83) | Map of fully qualified view ids keyed by view ids. |  |
+| [views](outputs.tf#L88) | View resources. |  |
 
 <!-- END TFDOC -->

modules/bigquery-dataset/outputs.tf

@@ -14,6 +14,21 @@
  * limitations under the License.
  */
 
+output "as_logging_destination" {
+  description = "Parameters to use this dataset as a log sink destination."
+  value = {
+    type   = "bigquery"
+    target = google_bigquery_dataset.default.id
+  }
+  depends_on = [
+    google_bigquery_dataset_access.domain,
+    google_bigquery_dataset_access.group_by_email,
+    google_bigquery_dataset_access.special_group,
+    google_bigquery_dataset_access.user_by_email,
+    google_bigquery_dataset_access.views
+  ]
+}
+
 output "dataset" {
   description = "Dataset resource."
   value       = google_bigquery_dataset.default

modules/folder/README.md

@@ -166,37 +166,27 @@ module "bucket" {
   id          = "bucket"
 }
 
+
 module "folder-sink" {
   source = "./fabric/modules/folder"
   parent = "folders/657104291943"
   name   = "my-folder"
   logging_sinks = {
     warnings = {
-      type             = "storage"
-      destination      = module.gcs.id
-      filter           = "severity=WARNING"
-      include_children = true
-      exclusions       = {}
+      destination = module.gcs.as_logging_destination
+      filter      = "severity=WARNING"
     }
     info = {
-      type             = "bigquery"
-      destination      = module.dataset.id
-      filter           = "severity=INFO"
-      include_children = true
-      exclusions       = {}
+      destination = module.dataset.as_logging_destination
+      filter      = "severity=INFO"
     }
     notice = {
-      type             = "pubsub"
-      destination      = module.pubsub.id
-      filter           = "severity=NOTICE"
-      include_children = true
-      exclusions       = {}
+      destination = module.pubsub.as_logging_destination
+      filter      = "severity=NOTICE"
     }
     debug = {
-      type             = "logging"
-      destination      = module.bucket.id
-      filter           = "severity=DEBUG"
-      include_children = true
+      destination = module.bucket.as_logging_destination
+      filter      = "severity=DEBUG"
       exclusions = {
         no-compute = "logName:compute"
       }
@@ -312,12 +302,12 @@ module "folder" {
 | [iam_additive_members](variables.tf#L85) | IAM additive bindings in {MEMBERS => [ROLE]} format. This might break if members are dynamic values. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [id](variables.tf#L92) | Folder ID in case you use folder_create=false. | <code>string</code> |  | <code>null</code> |
 | [logging_exclusions](variables.tf#L98) | Logging exclusions for this folder in the form {NAME -> FILTER}. | <code>map&#40;string&#41;</code> |  | <code>&#123;&#125;</code> |
-| [logging_sinks](variables.tf#L105) | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;  destination      &#61; string&#10;  type             &#61; string&#10;  filter           &#61; string&#10;  include_children &#61; bool&#10;  exclusions &#61; map&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [name](variables.tf#L126) | Folder name. | <code>string</code> |  | <code>null</code> |
-| [org_policies](variables.tf#L132) | Organization policies applied to this folder keyed by policy name. | <code title="map&#40;object&#40;&#123;&#10;  inherit_from_parent &#61; optional&#40;bool&#41; &#35; for list policies only.&#10;  reset               &#61; optional&#40;bool&#41;&#10;  allow &#61; optional&#40;object&#40;&#123;&#10;    all    &#61; optional&#40;bool&#41;&#10;    values &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;&#10;  deny &#61; optional&#40;object&#40;&#123;&#10;    all    &#61; optional&#40;bool&#41;&#10;    values &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;&#10;  enforce &#61; optional&#40;bool, true&#41; &#35; for boolean policies only.&#10;  rules &#61; optional&#40;list&#40;object&#40;&#123;&#10;    allow &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    deny &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    enforce &#61; optional&#40;bool, true&#41; &#35; for boolean policies only.&#10;    condition &#61; object&#40;&#123;&#10;      description &#61; optional&#40;string&#41;&#10;      expression  &#61; optional&#40;string&#41;&#10;      location    &#61; optional&#40;string&#41;&#10;      title       &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#10;  &#125;&#41;&#41;, &#91;&#93;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [org_policies_data_path](variables.tf#L172) | Path containing org policies in YAML format. | <code>string</code> |  | <code>null</code> |
-| [parent](variables.tf#L178) | Parent in folders/folder_id or organizations/org_id format. | <code>string</code> |  | <code>null</code> |
-| [tag_bindings](variables.tf#L188) | Tag bindings for this folder, in key => tag value id format. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
+| [logging_sinks](variables.tf#L105) | Logging sinks to create for this folder. | <code title="map&#40;object&#40;&#123;&#10;  bigquery_use_partitioned_table &#61; optional&#40;bool&#41;&#10;  description                    &#61; optional&#40;string&#41;&#10;  destination &#61; object&#40;&#123;&#10;    type   &#61; string&#10;    target &#61; string&#10;  &#125;&#41;&#10;  disabled         &#61; optional&#40;bool, false&#41;&#10;  exclusions       &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  filter           &#61; string&#10;  include_children &#61; optional&#40;bool, true&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [name](variables.tf#L137) | Folder name. | <code>string</code> |  | <code>null</code> |
+| [org_policies](variables.tf#L143) | Organization policies applied to this folder keyed by policy name. | <code title="map&#40;object&#40;&#123;&#10;  inherit_from_parent &#61; optional&#40;bool&#41; &#35; for list policies only.&#10;  reset               &#61; optional&#40;bool&#41;&#10;  allow &#61; optional&#40;object&#40;&#123;&#10;    all    &#61; optional&#40;bool&#41;&#10;    values &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;&#10;  deny &#61; optional&#40;object&#40;&#123;&#10;    all    &#61; optional&#40;bool&#41;&#10;    values &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;&#10;  enforce &#61; optional&#40;bool, true&#41; &#35; for boolean policies only.&#10;  rules &#61; optional&#40;list&#40;object&#40;&#123;&#10;    allow &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    deny &#61; optional&#40;object&#40;&#123;&#10;      all    &#61; optional&#40;bool&#41;&#10;      values &#61; optional&#40;list&#40;string&#41;&#41;&#10;    &#125;&#41;&#41;&#10;    enforce &#61; optional&#40;bool, true&#41; &#35; for boolean policies only.&#10;    condition &#61; object&#40;&#123;&#10;      description &#61; optional&#40;string&#41;&#10;      expression  &#61; optional&#40;string&#41;&#10;      location    &#61; optional&#40;string&#41;&#10;      title       &#61; optional&#40;string&#41;&#10;    &#125;&#41;&#10;  &#125;&#41;&#41;, &#91;&#93;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [org_policies_data_path](variables.tf#L183) | Path containing org policies in YAML format. | <code>string</code> |  | <code>null</code> |
+| [parent](variables.tf#L189) | Parent in folders/folder_id or organizations/org_id format. | <code>string</code> |  | <code>null</code> |
+| [tag_bindings](variables.tf#L199) | Tag bindings for this folder, in key => tag value id format. | <code>map&#40;string&#41;</code> |  | <code>null</code> |
 
 ## Outputs
 

modules/folder/logging.tf

@@ -22,19 +22,27 @@ locals {
     type => {
       for name, sink in var.logging_sinks :
       name => sink
-      if sink.type == type
+      if sink.destination.type == type
     }
   }
 }
 
 resource "google_logging_folder_sink" "sink" {
-  for_each = var.logging_sinks
-  name     = each.key
-  #description = "${each.key} (Terraform-managed)."
+  for_each         = var.logging_sinks
+  name             = each.key
+  description      = coalesce(each.value.description, "${each.key} (Terraform-managed).")
   folder           = local.folder.name
-  destination      = "${each.value.type}.googleapis.com/${each.value.destination}"
+  destination      = "${each.value.destination.type}.googleapis.com/${each.value.destination.target}"
   filter           = each.value.filter
   include_children = each.value.include_children
+  disabled         = each.value.disabled
+
+  dynamic "bigquery_options" {
+    for_each = each.value.bigquery_use_partitioned_table != null ? [""] : []
+    content {
+      use_partitioned_tables = each.value.bigquery_use_partitioned_table
+    }
+  }
 
   dynamic "exclusions" {
     for_each = each.value.exclusions
@@ -52,34 +60,38 @@ resource "google_logging_folder_sink" "sink" {
 
 resource "google_storage_bucket_iam_member" "gcs-sinks-binding" {
   for_each = local.sink_bindings["storage"]
-  bucket   = each.value.destination
+  bucket   = each.value.destination.target
   role     = "roles/storage.objectCreator"
   member   = google_logging_folder_sink.sink[each.key].writer_identity
 }
 
 resource "google_bigquery_dataset_iam_member" "bq-sinks-binding" {
   for_each   = local.sink_bindings["bigquery"]
-  project    = split("/", each.value.destination)[1]
-  dataset_id = split("/", each.value.destination)[3]
+  project    = split("/", each.value.destination.target)[1]
+  dataset_id = split("/", each.value.destination.target)[3]
   role       = "roles/bigquery.dataEditor"
   member     = google_logging_folder_sink.sink[each.key].writer_identity
 }
 
 resource "google_pubsub_topic_iam_member" "pubsub-sinks-binding" {
   for_each = local.sink_bindings["pubsub"]
-  project  = split("/", each.value.destination)[1]
-  topic    = split("/", each.value.destination)[3]
+  project  = split("/", each.value.destination.target)[1]
+  topic    = split("/", each.value.destination.target)[3]
   role     = "roles/pubsub.publisher"
   member   = google_logging_folder_sink.sink[each.key].writer_identity
 }
 
 resource "google_project_iam_member" "bucket-sinks-binding" {
   for_each = local.sink_bindings["logging"]
-  project  = split("/", each.value.destination)[1]
+  project  = split("/", each.value.destination.target)[1]
   role     = "roles/logging.bucketWriter"
   member   = google_logging_folder_sink.sink[each.key].writer_identity
-  # TODO(jccb): use a condition to limit writer-identity only to this
-  # bucket
+
+  condition {
+    title       = "${each.key} bucket writer"
+    description = "Grants bucketWriter to ${google_logging_folder_sink.sink[each.key].writer_identity} used by log sink ${each.key} on ${local.folder.id}"
+    expression  = "resource.name.endsWith('${each.value.destination.target}')"
+  }
 }
 
 resource "google_logging_folder_exclusion" "logging-exclusion" {

modules/folder/variables.tf

@@ -105,22 +105,33 @@ variable "logging_exclusions" {
 variable "logging_sinks" {
   description = "Logging sinks to create for this folder."
   type = map(object({
-    destination      = string
-    type             = string
+    bigquery_use_partitioned_table = optional(bool)
+    description                    = optional(string)
+    destination = object({
+      type   = string
+      target = string
+    })
+    disabled         = optional(bool, false)
+    exclusions       = optional(map(string), {})
     filter           = string
-    include_children = bool
-    # TODO exclusions also support description and disabled
-    exclusions = map(string)
+    include_children = optional(bool, true)
   }))
+  default  = {}
+  nullable = false
   validation {
     condition = alltrue([
-      for k, v in(var.logging_sinks == null ? {} : var.logging_sinks) :
-      contains(["bigquery", "logging", "pubsub", "storage"], v.type)
+      for k, v in var.logging_sinks :
+      contains(["bigquery", "logging", "pubsub", "storage"], v.destination.type)
     ])
-    error_message = "Type must be one of 'bigquery', 'logging', 'pubsub', 'storage'."
+    error_message = "Destination type must be one of 'bigquery', 'logging', 'pubsub', 'storage'."
+  }
+  validation {
+    condition = alltrue([
+      for k, v in var.logging_sinks :
+      v.bigquery_use_partitioned_table != true || v.destination.type == "bigquery"
+    ])
+    error_message = "Can only set bigquery_use_partitioned_table when destination type is `bigquery`."
   }
-  default  = {}
-  nullable = false
 }
 
 variable "name" {

modules/gcs/README.md

@@ -134,11 +134,11 @@ module "bucket-gcs-notification" {
 
 | name | description | sensitive |
 |---|---|:---:|
-| [bucket](outputs.tf#L17) | Bucket resource. |  |
-| [id](outputs.tf#L28) | Bucket ID (same as name). |  |
-| [name](outputs.tf#L37) | Bucket name. |  |
-| [notification](outputs.tf#L46) | GCS Notification self link. |  |
-| [topic](outputs.tf#L51) | Topic ID used by GCS. |  |
-| [url](outputs.tf#L56) | Bucket URL. |  |
+| [as_logging_destination](outputs.tf#L17) | Parameters to use this bucket as a log sink destination. |  |
+| [bucket](outputs.tf#L29) | Bucket resource. |  |
+| [name](outputs.tf#L34) | Bucket name. |  |
+| [notification](outputs.tf#L43) | GCS Notification self link. |  |
+| [topic](outputs.tf#L48) | Topic ID used by GCS. |  |
+| [url](outputs.tf#L53) | Bucket URL. |  |
 
 <!-- END TFDOC -->

modules/gcs/outputs.tf

@@ -14,26 +14,23 @@
  * limitations under the License.
  */
 
-output "bucket" {
-  description = "Bucket resource."
-  value       = google_storage_bucket.bucket
-}
-
-# We add `id` as an alias to `name` to simplify log sink handling.
-# Since all other log destinations (pubsub, logging-bucket, bigquery)
-# have an id output, it is convenient to have in this module too to
-# handle all log destination as homogeneous objects (i.e. you can
-# assume any valid log destination has an `id` output).
-
-output "id" {
-  description = "Bucket ID (same as name)."
-  value       = "${local.prefix}${lower(var.name)}"
+output "as_logging_destination" {
+  description = "Parameters to use this bucket as a log sink destination."
+  value = {
+    type   = "storage"
+    target = "${local.prefix}${lower(var.name)}"
+  }
   depends_on = [
     google_storage_bucket.bucket,
     google_storage_bucket_iam_binding.bindings
   ]
 }
 
+output "bucket" {
+  description = "Bucket resource."
+  value       = google_storage_bucket.bucket
+}
+
 output "name" {
   description = "Bucket name."
   value       = "${local.prefix}${lower(var.name)}"

modules/logging-bucket/README.md

@@ -56,6 +56,7 @@ module "bucket-default" {
 
 | name | description | sensitive |
 |---|---|:---:|
-| [id](outputs.tf#L17) | ID of the created bucket. |  |
+| [as_logging_destination](outputs.tf#L17) | Parameters to use this bucket as a log sink destination. |  |
+| [id](outputs.tf#L30) | ID of the created bucket. |  |
 
 <!-- END TFDOC -->

modules/logging-bucket/outputs.tf

@@ -14,6 +14,19 @@
  * limitations under the License.
  */
 
+output "as_logging_destination" {
+  description = "Parameters to use this bucket as a log sink destination."
+  value = {
+    type = "logging"
+    target = try(
+      google_logging_project_bucket_config.bucket.0.id,
+      google_logging_folder_bucket_config.bucket.0.id,
+      google_logging_organization_bucket_config.bucket.0.id,
+      google_logging_billing_account_bucket_config.bucket.0.id,
+    )
+  }
+}
+
 output "id" {
   description = "ID of the created bucket."
   value = try(