1414
1515# tfdoc:file:description Orchestration Cloud Composer definition.
1616
17+ locals {
18+ env_variables =
19+ BQ_LOCATION = var.location
20+ DATA_CAT_TAGS = try (jsonencode (module. common-datacatalog . tags ), " {}"
21+ DF_KMS_KEY = try (var. service_encryption_keys . dataflow , " "
22+ DRP_PRJ = module.drop- project.project_id
23+ DRP_BQ = module.drop- bq- 0 .dataset_id
24+ DRP_GCS = module.drop- cs- 0 .url
25+ DRP_PS = module.drop- ps- 0 .id
26+ DWH_LAND_PRJ = module.dwh- lnd- project.project_id
27+ DWH_LAND_BQ_DATASET = module.dwh- lnd- bq- 0 .dataset_id
28+ DWH_LAND_GCS = module.dwh- lnd- cs- 0 .url
29+ DWH_CURATED_PRJ = module.dwh- cur- project.project_id
30+ DWH_CURATED_BQ_DATASET = module.dwh- cur- bq- 0 .dataset_id
31+ DWH_CURATED_GCS = module.dwh- cur- cs- 0 .url
32+ DWH_CONFIDENTIAL_PRJ = module.dwh- conf- project.project_id
33+ DWH_CONFIDENTIAL_BQ_DATASET = module.dwh- conf- bq- 0 .dataset_id
34+ DWH_CONFIDENTIAL_GCS = module.dwh- conf- cs- 0 .url
35+ GCP_REGION = var.region
36+ LOD_PRJ = module.load- project.project_id
37+ LOD_GCS_STAGING = module.load- cs- df- 0 .url
38+ LOD_NET_VPC = local.load_vpc
39+ LOD_NET_SUBNET = local.load_subnet
40+ LOD_SA_DF = module.load- sa- df- 0 .email
41+ ORC_PRJ = module.orch- project.project_id
42+ ORC_GCS = module.orch- cs- 0 .url
43+ TRF_PRJ = module.transf- project.project_id
44+ TRF_GCS_STAGING = module.transf- cs- df- 0 .url
45+ TRF_NET_VPC = local.transf_vpc
46+ TRF_NET_SUBNET = local.transf_subnet
47+ TRF_SA_DF = module.transf- sa- df- 0 .email
48+ TRF_SA_BQ = module.transf- sa- bq- 0 .email
49+ }
50+ }
1751module "orch-sa-cmp-0" {
1852 source = " ../../../modules/iam-service-account"
1953 project_id = . orch-project . project_id
@@ -27,21 +61,51 @@ module "orch-sa-cmp-0" {
2761}
2862
2963resource "google_composer_environment" "orch-cmp-0" {
30- provider = google - beta
31- project = . orch-project . project_id
32- name = " ${ var . prefix } -orc-cmp-0"
33- region = . region
64+ count = var . composer_config . disable_deployment == true ? 0 : 1
65+ project = . orch-project . project_id
66+ name = " ${ var . prefix } -orc-cmp-0"
67+ region = . region
3468 config {
35- node_count = . composer_config . node_count
69+ software_config {
70+ airflow_config_overrides = try (var. composer_config . software_config . airflow_config_overrides , null )
71+ pypi_packages = try (var. composer_config . software_config . pypi_packages , null )
72+ env_variables = merge (try (var. composer_config . software_config . env_variables , null ), local. env_variables )
73+ image_version = try (var. composer_config . software_config . image_version , null )
74+ }
75+ dynamic "workloads_config" {
76+ for_each = try (var. composer_config . workloads_config , null ) != null ? { 1 = 1 } : {})
77+
78+ content {
79+ scheduler {
80+ cpu = try (var. composer_config . workloads_config . scheduler . cpu , null )
81+ memory_gb = try (var. composer_config . workloads_config . scheduler . memory_gb , null )
82+ storage_gb = try (var. composer_config . workloads_config . scheduler . storage_gb , null )
83+ count = try (var. composer_config . workloads_config . scheduler . count , null )
84+ }
85+ web_server {
86+ cpu = try (var. composer_config . workloads_config . web_server . cpu , null )
87+ memory_gb = try (var. composer_config . workloads_config . web_server . memory_gb , null )
88+ storage_gb = try (var. composer_config . workloads_config . web_server . storage_gb , null )
89+ }
90+ worker {
91+ cpu = try (var. composer_config . workloads_config . worker . cpu , null )
92+ memory_gb = try (var. composer_config . workloads_config . worker . memory_gb , null )
93+ storage_gb = try (var. composer_config . workloads_config . worker . storage_gb , null )
94+ min_count = try (var. composer_config . workloads_config . worker . min_count , null )
95+ max_count = try (var. composer_config . workloads_config . worker . max_count , null )
96+ }
97+ }
98+ }
99+
100+ environment_size = . composer_config . environment_size
101+
36102 node_config {
37- zone = " ${ var . region } -b"
38- service_account = . orch-sa-cmp-0 . email
39103 network = . orch_vpc
40104 subnetwork = . orch_subnet
41- tags = " composer-worker" " http-server" " https-server"
42- enable_ip_masq_agent = true
105+ service_account = . orch-sa-cmp-0 . email
106+ enable_ip_masq_agent = " true"
107+ tags = " composer-worker"
43108 ip_allocation_policy {
44- use_ip_aliases = " true"
45109 cluster_secondary_range_name = try (
46110 var. network_config . composer_secondary_ranges . pods , " pods"
47111 )
@@ -58,80 +122,20 @@ resource "google_composer_environment" "orch-cmp-0" {
58122 master_ipv4_cidr_block = try (
59123 var. network_config . composer_ip_ranges . gke_master , " 10.20.11.0/28"
60124 )
61- web_server_ipv4_cidr_block = try (
62- var. network_config . composer_ip_ranges . web_server , " 10.20.11.16/28"
63- )
64125 }
65- software_config {
66- image_version = . composer_config . airflow_version
67- env_variables = merge (
68- var. composer_config . env_variables , {
69- BQ_LOCATION = var.location
70- DATA_CAT_TAGS = try (jsonencode (module. common-datacatalog . tags ), " {}"
71- DF_KMS_KEY = try (var. service_encryption_keys . dataflow , " "
72- DRP_PRJ = module.drop- project.project_id
73- DRP_BQ = module.drop- bq- 0 .dataset_id
74- DRP_GCS = module.drop- cs- 0 .url
75- DRP_PS = module.drop- ps- 0 .id
76- DWH_LAND_PRJ = module.dwh- lnd- project.project_id
77- DWH_LAND_BQ_DATASET = module.dwh- lnd- bq- 0 .dataset_id
78- DWH_LAND_GCS = module.dwh- lnd- cs- 0 .url
79- DWH_CURATED_PRJ = module.dwh- cur- project.project_id
80- DWH_CURATED_BQ_DATASET = module.dwh- cur- bq- 0 .dataset_id
81- DWH_CURATED_GCS = module.dwh- cur- cs- 0 .url
82- DWH_CONFIDENTIAL_PRJ = module.dwh- conf- project.project_id
83- DWH_CONFIDENTIAL_BQ_DATASET = module.dwh- conf- bq- 0 .dataset_id
84- DWH_CONFIDENTIAL_GCS = module.dwh- conf- cs- 0 .url
85- DWH_PLG_PRJ = module.dwh- plg- project.project_id
86- DWH_PLG_BQ_DATASET = module.dwh- plg- bq- 0 .dataset_id
87- DWH_PLG_GCS = module.dwh- plg- cs- 0 .url
88- GCP_REGION = var.region
89- LOD_PRJ = module.load- project.project_id
90- LOD_GCS_STAGING = module.load- cs- df- 0 .url
91- LOD_NET_VPC = local.load_vpc
92- LOD_NET_SUBNET = local.load_subnet
93- LOD_SA_DF = module.load- sa- df- 0 .email
94- ORC_PRJ = module.orch- project.project_id
95- ORC_GCS = module.orch- cs- 0 .url
96- TRF_PRJ = module.transf- project.project_id
97- TRF_GCS_STAGING = module.transf- cs- df- 0 .url
98- TRF_NET_VPC = local.transf_vpc
99- TRF_NET_SUBNET = local.transf_subnet
100- TRF_SA_DF = module.transf- sa- df- 0 .email
101- TRF_SA_BQ = module.transf- sa- bq- 0 .email
102- }
103- )
104- }
105-
106126 dynamic "encryption_config" {
107127 for_each =
108- try (local . service_encryption_keys . composer != null , false )
128+ try (var . service_encryption_keys [ var . region ], null ) != null
109129 ? { 1 = 1 }
110130 : {}
111131 )
112132 content {
113- kms_key_name = try (local . service_encryption_keys . composer , null )
133+ kms_key_name = try (var . service_encryption_keys [ var . region ] , null )
114134 }
115135 }
116-
117- # dynamic "web_server_network_access_control" {
118- # for_each = toset(
119- # var.network_config.web_server_network_access_control == null
120- # ? []
121- # : [var.network_config.web_server_network_access_control]
122- # )
123- # content {
124- # dynamic "allowed_ip_range" {
125- # for_each = toset(web_server_network_access_control.key)
126- # content {
127- # value = allowed_ip_range.key
128- # }
129- # }
130- # }
131- # }
132-
133136 }
134137 depends_on =
135138 google_project_iam_member . shared_vpc ,
139+ module . orch-project
136140 ]
137141}
0 commit comments