juliodiez · juliodiez · Jan 16, 2023 · Jan 10, 2023 · Jan 10, 2023 · Jan 11, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file.
 
 ### BLUEPRINTS
 
+- [[#1081](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1081)] Apigee hybrid on GKE ([apichick](https://github.com/apichick)) <!-- 2023-01-05 08:23:33+00:00 -->
+- [[#1082](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1082)] Fixes in Apigee Bigquery Analytics blueprint ([apichick](https://github.com/apichick)) <!-- 2023-01-04 16:42:50+00:00 -->
 - [[#1071](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1071)] Moved apigee bigquery analytics blueprint, added apigee network patterns ([apichick](https://github.com/apichick)) <!-- 2022-12-23 15:16:45+00:00 -->
 - [[#1073](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1073)] Allow setting no ranges in firewall module custom rules ([ludoo](https://github.com/ludoo)) <!-- 2022-12-23 08:03:31+00:00 -->
 - [[#1072](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1072)] **incompatible change:** Add gc_policy to Bigtable module, bump provider versions to 4.47 ([iht](https://github.com/iht)) <!-- 2022-12-22 23:58:08+00:00 -->
@@ -18,17 +20,25 @@ All notable changes to this project will be documented in this file.
 
 ### DOCUMENTATION
 
+- [[#1084](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1084)] Fixes in Apigee blueprints README files ([apichick](https://github.com/apichick)) <!-- 2023-01-05 11:00:46+00:00 -->
+- [[#1081](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1081)] Apigee hybrid on GKE ([apichick](https://github.com/apichick)) <!-- 2023-01-05 08:23:33+00:00 -->
+- [[#1074](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1074)] Adding new section for Authentication issues ([agutta](https://github.com/agutta)) <!-- 2022-12-29 15:50:23+00:00 -->
 - [[#1071](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1071)] Moved apigee bigquery analytics blueprint, added apigee network patterns ([apichick](https://github.com/apichick)) <!-- 2022-12-23 15:16:45+00:00 -->
 - [[#1057](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1057)] Adding new file FAQ and an image ([agutta](https://github.com/agutta)) <!-- 2022-12-22 14:00:22+00:00 -->
 
 ### FAST
 
+- [[#1085](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1085)] fix restricted services not being added to the perimeter configurations ([drebes](https://github.com/drebes)) <!-- 2023-01-06 12:25:31+00:00 -->
 - [[#1057](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1057)] Adding new file FAQ and an image ([agutta](https://github.com/agutta)) <!-- 2022-12-22 14:00:22+00:00 -->
 - [[#1054](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1054)] FAST: fix typo in bootstrap stage README ([agutta](https://github.com/agutta)) <!-- 2022-12-16 16:00:00+00:00 -->
 - [[#1051](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1051)] FAST: add instructions for billing export to stage 0 README ([KPRepos](https://github.com/KPRepos)) <!-- 2022-12-15 08:53:57+00:00 -->
 
 ### MODULES
 
+- [[#1078](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1078)] Fixed delete_rule in compute-mig module for stateful disks ([rosmo](https://github.com/rosmo)) <!-- 2023-01-04 08:14:40+00:00 -->
+- [[#1080](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1080)] Added device_name field to compute-vm attached_disks parameter  ([rosmo](https://github.com/rosmo)) <!-- 2023-01-03 20:53:48+00:00 -->
+- [[#1079](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1079)] Reorder org policy rules ([juliocc](https://github.com/juliocc)) <!-- 2023-01-03 16:11:29+00:00 -->
+- [[#1075](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1075)] **incompatible change:** Add cluster replicas to Bigtable module. ([iht](https://github.com/iht)) <!-- 2022-12-30 10:39:38+00:00 -->
 - [[#1073](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1073)] Allow setting no ranges in firewall module custom rules ([ludoo](https://github.com/ludoo)) <!-- 2022-12-23 08:03:31+00:00 -->
 - [[#1072](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1072)] **incompatible change:** Add gc_policy to Bigtable module, bump provider versions to 4.47 ([iht](https://github.com/iht)) <!-- 2022-12-22 23:58:08+00:00 -->
 - [[#1070](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1070)] Fix MIG health check variable ([ludoo](https://github.com/ludoo)) <!-- 2022-12-22 17:12:17+00:00 -->
@@ -43,6 +53,7 @@ All notable changes to this project will be documented in this file.
 
 ### TOOLS
 
+- [[#1091](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1091)] Fix check_documentation output ([juliocc](https://github.com/juliocc)) <!-- 2023-01-12 14:43:13+00:00 -->
 - [[#1053](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1053)] Extend inventory-based testing to examples ([juliocc](https://github.com/juliocc)) <!-- 2022-12-18 19:50:34+00:00 -->
 
 ## [19.0.0] - 2022-12-13

diff --git a/blueprints/data-solutions/data-platform-foundations/03-composer.tf b/blueprints/data-solutions/data-platform-foundations/03-composer.tf
@@ -14,6 +14,40 @@
 
 # tfdoc:file:description Orchestration Cloud Composer definition.
 
+locals {
+  env_variables = {
+    BQ_LOCATION                 = var.location
+    DATA_CAT_TAGS               = try(jsonencode(module.common-datacatalog.tags), "{}")
+    DF_KMS_KEY                  = try(var.service_encryption_keys.dataflow, "")
+    DRP_PRJ                     = module.drop-project.project_id
+    DRP_BQ                      = module.drop-bq-0.dataset_id
+    DRP_GCS                     = module.drop-cs-0.url
+    DRP_PS                      = module.drop-ps-0.id
+    DWH_LAND_PRJ                = module.dwh-lnd-project.project_id
+    DWH_LAND_BQ_DATASET         = module.dwh-lnd-bq-0.dataset_id
+    DWH_LAND_GCS                = module.dwh-lnd-cs-0.url
+    DWH_CURATED_PRJ             = module.dwh-cur-project.project_id
+    DWH_CURATED_BQ_DATASET      = module.dwh-cur-bq-0.dataset_id
+    DWH_CURATED_GCS             = module.dwh-cur-cs-0.url
+    DWH_CONFIDENTIAL_PRJ        = module.dwh-conf-project.project_id
+    DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
+    DWH_CONFIDENTIAL_GCS        = module.dwh-conf-cs-0.url
+    GCP_REGION                  = var.region
+    LOD_PRJ                     = module.load-project.project_id
+    LOD_GCS_STAGING             = module.load-cs-df-0.url
+    LOD_NET_VPC                 = local.load_vpc
+    LOD_NET_SUBNET              = local.load_subnet
+    LOD_SA_DF                   = module.load-sa-df-0.email
+    ORC_PRJ                     = module.orch-project.project_id
+    ORC_GCS                     = module.orch-cs-0.url
+    TRF_PRJ                     = module.transf-project.project_id
+    TRF_GCS_STAGING             = module.transf-cs-df-0.url
+    TRF_NET_VPC                 = local.transf_vpc
+    TRF_NET_SUBNET              = local.transf_subnet
+    TRF_SA_DF                   = module.transf-sa-df-0.email
+    TRF_SA_BQ                   = module.transf-sa-bq-0.email
+  }
+}
 module "orch-sa-cmp-0" {
   source       = "../../../modules/iam-service-account"
   project_id   = module.orch-project.project_id
@@ -27,21 +61,51 @@ module "orch-sa-cmp-0" {
 }
 
 resource "google_composer_environment" "orch-cmp-0" {
-  provider = google-beta
-  project  = module.orch-project.project_id
-  name     = "${var.prefix}-orc-cmp-0"
-  region   = var.region
+  count   = var.composer_config.disable_deployment == true ? 0 : 1
+  project = module.orch-project.project_id
+  name    = "${var.prefix}-orc-cmp-0"
+  region  = var.region
   config {
-    node_count = var.composer_config.node_count
+    software_config {
+      airflow_config_overrides = try(var.composer_config.software_config.airflow_config_overrides, null)
+      pypi_packages            = try(var.composer_config.software_config.pypi_packages, null)
+      env_variables            = merge(try(var.composer_config.software_config.env_variables, null), local.env_variables)
+      image_version            = try(var.composer_config.software_config.image_version, null)
+    }
+    dynamic "workloads_config" {
+      for_each = (try(var.composer_config.workloads_config, null) != null ? { 1 = 1 } : {})
+
+      content {
+        scheduler {
+          cpu        = try(var.composer_config.workloads_config.scheduler.cpu, null)
+          memory_gb  = try(var.composer_config.workloads_config.scheduler.memory_gb, null)
+          storage_gb = try(var.composer_config.workloads_config.scheduler.storage_gb, null)
+          count      = try(var.composer_config.workloads_config.scheduler.count, null)
+        }
+        web_server {
+          cpu        = try(var.composer_config.workloads_config.web_server.cpu, null)
+          memory_gb  = try(var.composer_config.workloads_config.web_server.memory_gb, null)
+          storage_gb = try(var.composer_config.workloads_config.web_server.storage_gb, null)
+        }
+        worker {
+          cpu        = try(var.composer_config.workloads_config.worker.cpu, null)
+          memory_gb  = try(var.composer_config.workloads_config.worker.memory_gb, null)
+          storage_gb = try(var.composer_config.workloads_config.worker.storage_gb, null)
+          min_count  = try(var.composer_config.workloads_config.worker.min_count, null)
+          max_count  = try(var.composer_config.workloads_config.worker.max_count, null)
+        }
+      }
+    }
+
+    environment_size = var.composer_config.environment_size
+
     node_config {
-      zone                 = "${var.region}-b"
-      service_account      = module.orch-sa-cmp-0.email
       network              = local.orch_vpc
       subnetwork           = local.orch_subnet
-      tags                 = ["composer-worker", "http-server", "https-server"]
-      enable_ip_masq_agent = true
+      service_account      = module.orch-sa-cmp-0.email
+      enable_ip_masq_agent = "true"
+      tags                 = ["composer-worker"]
       ip_allocation_policy {
-        use_ip_aliases = "true"
         cluster_secondary_range_name = try(
           var.network_config.composer_secondary_ranges.pods, "pods"
         )
@@ -58,80 +122,20 @@ resource "google_composer_environment" "orch-cmp-0" {
       master_ipv4_cidr_block = try(
         var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28"
       )
-      web_server_ipv4_cidr_block = try(
-        var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28"
-      )
     }
-    software_config {
-      image_version = var.composer_config.airflow_version
-      env_variables = merge(
-        var.composer_config.env_variables, {
-          BQ_LOCATION                 = var.location
-          DATA_CAT_TAGS               = try(jsonencode(module.common-datacatalog.tags), "{}")
-          DF_KMS_KEY                  = try(var.service_encryption_keys.dataflow, "")
-          DRP_PRJ                     = module.drop-project.project_id
-          DRP_BQ                      = module.drop-bq-0.dataset_id
-          DRP_GCS                     = module.drop-cs-0.url
-          DRP_PS                      = module.drop-ps-0.id
-          DWH_LAND_PRJ                = module.dwh-lnd-project.project_id
-          DWH_LAND_BQ_DATASET         = module.dwh-lnd-bq-0.dataset_id
-          DWH_LAND_GCS                = module.dwh-lnd-cs-0.url
-          DWH_CURATED_PRJ             = module.dwh-cur-project.project_id
-          DWH_CURATED_BQ_DATASET      = module.dwh-cur-bq-0.dataset_id
-          DWH_CURATED_GCS             = module.dwh-cur-cs-0.url
-          DWH_CONFIDENTIAL_PRJ        = module.dwh-conf-project.project_id
-          DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
-          DWH_CONFIDENTIAL_GCS        = module.dwh-conf-cs-0.url
-          DWH_PLG_PRJ                 = module.dwh-plg-project.project_id
-          DWH_PLG_BQ_DATASET          = module.dwh-plg-bq-0.dataset_id
-          DWH_PLG_GCS                 = module.dwh-plg-cs-0.url
-          GCP_REGION                  = var.region
-          LOD_PRJ                     = module.load-project.project_id
-          LOD_GCS_STAGING             = module.load-cs-df-0.url
-          LOD_NET_VPC                 = local.load_vpc
-          LOD_NET_SUBNET              = local.load_subnet
-          LOD_SA_DF                   = module.load-sa-df-0.email
-          ORC_PRJ                     = module.orch-project.project_id
-          ORC_GCS                     = module.orch-cs-0.url
-          TRF_PRJ                     = module.transf-project.project_id
-          TRF_GCS_STAGING             = module.transf-cs-df-0.url
-          TRF_NET_VPC                 = local.transf_vpc
-          TRF_NET_SUBNET              = local.transf_subnet
-          TRF_SA_DF                   = module.transf-sa-df-0.email
-          TRF_SA_BQ                   = module.transf-sa-bq-0.email
-        }
-      )
-    }
-
     dynamic "encryption_config" {
       for_each = (
-        try(local.service_encryption_keys.composer != null, false)
+        try(var.service_encryption_keys[var.region], null) != null
         ? { 1 = 1 }
         : {}
       )
       content {
-        kms_key_name = try(local.service_encryption_keys.composer, null)
+        kms_key_name = try(var.service_encryption_keys[var.region], null)
       }
     }
-
-    # dynamic "web_server_network_access_control" {
-    #   for_each = toset(
-    #     var.network_config.web_server_network_access_control == null
-    #     ? []
-    #     : [var.network_config.web_server_network_access_control]
-    #   )
-    #   content {
-    #     dynamic "allowed_ip_range" {
-    #       for_each = toset(web_server_network_access_control.key)
-    #       content {
-    #         value = allowed_ip_range.key
-    #       }
-    #     }
-    #   }
-    # }
-
   }
   depends_on = [
     google_project_iam_member.shared_vpc,
+    module.orch-project
   ]
 }
diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
@@ -54,6 +54,9 @@ module "orch-project" {
     "roles/bigquery.jobUser" = [
       module.orch-sa-cmp-0.iam_email,
     ]
+    "roles/composer.ServiceAgentV2Ext" = [
+      "serviceAccount:${module.orch-project.service_accounts.robots.composer}"
+    ]
     "roles/composer.worker" = [
       module.orch-sa-cmp-0.iam_email
     ]
@@ -67,11 +70,6 @@ module "orch-project" {
     "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
   }
   oslogin = false
-  org_policies = {
-    "constraints/compute.requireOsLogin" = {
-      enforce = false
-    }
-  }
   services = concat(var.project_services, [
     "artifactregistry.googleapis.com",
     "bigquery.googleapis.com",

diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
@@ -30,21 +30,6 @@ locals {
       "roles/storage.objectViewer",
     ]
   }
-  dwh_plg_group_iam = {
-    (local.groups.data-engineers) = [
-      "roles/bigquery.dataEditor",
-      "roles/storage.admin",
-    ],
-    (local.groups.data-analysts) = [
-      "roles/bigquery.dataEditor",
-      "roles/bigquery.jobUser",
-      "roles/bigquery.metadataViewer",
-      "roles/bigquery.user",
-      "roles/datacatalog.viewer",
-      "roles/datacatalog.tagTemplateViewer",
-      "roles/storage.objectAdmin",
-    ]
-  }
   dwh_lnd_iam = {
     "roles/bigquery.dataOwner" = [
       module.load-sa-df-0.iam_email,
@@ -140,21 +125,6 @@ module "dwh-conf-project" {
   }
 }
 
-module "dwh-plg-project" {
-  source          = "../../../modules/project"
-  parent          = var.folder_id
-  billing_account = var.billing_account_id
-  prefix          = var.prefix
-  name            = "dwh-plg${local.project_suffix}"
-  group_iam       = local.dwh_plg_group_iam
-  iam             = {}
-  services        = local.dwh_services
-  service_encryption_key_ids = {
-    bq      = [try(local.service_encryption_keys.bq, null)]
-    storage = [try(local.service_encryption_keys.storage, null)]
-  }
-}
-
 # Bigquery
 
 module "dwh-lnd-bq-0" {
@@ -181,14 +151,6 @@ module "dwh-conf-bq-0" {
   encryption_key = try(local.service_encryption_keys.bq, null)
 }
 
-module "dwh-plg-bq-0" {
-  source         = "../../../modules/bigquery-dataset"
-  project_id     = module.dwh-plg-project.project_id
-  id             = "${replace(var.prefix, "-", "_")}_dwh_plg_bq_0"
-  location       = var.location
-  encryption_key = try(local.service_encryption_keys.bq, null)
-}
-
 # Cloud storage
 
 module "dwh-lnd-cs-0" {
@@ -223,14 +185,3 @@ module "dwh-conf-cs-0" {
   encryption_key = try(local.service_encryption_keys.storage, null)
   force_destroy  = var.data_force_destroy
 }
-
-module "dwh-plg-cs-0" {
-  source         = "../../../modules/gcs"
-  project_id     = module.dwh-plg-project.project_id
-  prefix         = var.prefix
-  name           = "dwh-plg-cs-0"
-  location       = var.location
-  storage_class  = "MULTI_REGIONAL"
-  encryption_key = try(local.service_encryption_keys.storage, null)
-  force_destroy  = var.data_force_destroy
-}