Added functionality to write tfvars files

.github/workflows/terraform.yaml

@@ -10,7 +10,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: hashicorp/setup-terraform@v3
       with:
-        terraform_version: "1.5.7" # Cloud Shell version
+        terraform_version: "1.8.0"
 
     - name: terraform fmt
       id: fmt

platforms/gke/base/_shared_config/terraform_variables.tf

@@ -21,6 +21,12 @@ locals {
   terraform_bucket_name = "${var.terraform_project_id}-${local.unique_identifier_prefix}-terraform"
 }
 
+variable "create_terraform_bucket" {
+  default     = true
+  description = "Create the Google Cloud Storage Terraform bucket."
+  type        = string
+}
+
 variable "terraform_project_id" {
   description = "The GCP project where terraform will be run"
   type        = string
@@ -31,8 +37,8 @@ variable "terraform_project_id" {
   }
 }
 
-variable "create_terraform_bucket" {
+variable "terraform_write_tfvars" {
   default     = true
-  description = "Create the Google Cloud Storage Terraform bucket"
+  description = "Write the configured values to the tfvars configuration files."
   type        = string
 }

platforms/gke/base/core/initialize/_configmanagement.auto.tfvars

@@ -0,0 +1 @@
+../../_shared_config/configmanagement.auto.tfvars

platforms/gke/base/core/initialize/_configmanagement_variables.tf

@@ -0,0 +1 @@
+../../_shared_config/configmanagement_variables.tf

platforms/gke/base/core/initialize/_networking.auto.tfvars

@@ -0,0 +1 @@
+../../_shared_config/networking.auto.tfvars

platforms/gke/base/core/initialize/_networking_variables.tf

@@ -0,0 +1 @@
+../../_shared_config/networking_variables.tf

platforms/gke/base/core/initialize/_workloads.auto.tfvars

@@ -0,0 +1 @@
+../../_shared_config/workloads.auto.tfvars

platforms/gke/base/core/initialize/_workloads_variables.tf

@@ -0,0 +1 @@
+../../_shared_config/workloads_variables.tf

platforms/gke/base/core/initialize/local_file.tf

@@ -20,6 +20,8 @@ locals {
   core_backend_directories = toset([for _, version_file in local.core_versions_files : trimprefix(trimsuffix(version_file, "/versions.tf"), "../")])
   core_versions_files      = flatten([for _, file in flatten(fileset(local.base_directory, "core/**/versions.tf")) : file])
 
+  shared_config_folder = "${path.module}/../../_shared_config"
+
   use_case_backend_directories = var.initialize_backend_use_case_name != null ? toset([for _, version_file in local.use_case_versions_files : trimprefix(trimsuffix(dirname(version_file), "/versions.tf"), "../")]) : []
   use_case_versions_files      = var.initialize_backend_use_case_name != null ? flatten([for _, file in flatten(fileset("${local.base_directory}/use-cases", "${var.initialize_backend_use_case_name}/**/versions.tf")) : file]) : []
 }
@@ -37,6 +39,126 @@ resource "local_file" "core_backend_tf" {
   filename        = "${local.base_directory}/${each.key}/backend.tf"
 }
 
+resource "local_file" "shared_config_cluster_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      cluster_auto_monitoring_config_scope                 = var.cluster_auto_monitoring_config_scope
+      cluster_binary_authorization_evaluation_mode         = var.cluster_binary_authorization_evaluation_mode
+      cluster_confidential_nodes_enabled                   = var.cluster_confidential_nodes_enabled
+      cluster_database_encryption_key_name                 = var.cluster_database_encryption_key_name
+      cluster_database_encryption_state                    = var.cluster_database_encryption_state
+      cluster_enable_private_endpoint                      = var.cluster_enable_private_endpoint
+      cluster_gateway_api_config_channel                   = var.cluster_gateway_api_config_channel
+      cluster_gpu_driver_version                           = var.cluster_gpu_driver_version
+      cluster_master_global_access_enabled                 = var.cluster_master_global_access_enabled
+      cluster_master_ipv4_cidr_block                       = var.cluster_master_ipv4_cidr_block
+      cluster_node_auto_provisioning_enabled               = var.cluster_node_auto_provisioning_enabled
+      cluster_node_auto_provisioning_resource_limits       = var.cluster_node_auto_provisioning_resource_limits
+      cluster_node_pool_default_service_account_id         = var.cluster_node_pool_default_service_account_id
+      cluster_node_pool_default_service_account_project_id = var.cluster_node_pool_default_service_account_project_id
+      cluster_private_endpoint_subnetwork                  = var.cluster_private_endpoint_subnetwork
+      cluster_project_id                                   = var.cluster_project_id
+      cluster_region                                       = var.cluster_region
+      cluster_system_node_pool_machine_type                = var.cluster_system_node_pool_machine_type
+      cluster_use_connect_gateway                          = var.cluster_use_connect_gateway
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/cluster.auto.tfvars"
+}
+
+resource "local_file" "shared_config_configmanagement_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      configmanagement_git_credentials = var.configmanagement_git_credentials
+      configmanagement_policy_dir      = var.configmanagement_policy_dir
+      configmanagement_prevent_drift   = var.configmanagement_prevent_drift
+      configmanagement_sync_branch     = var.configmanagement_sync_branch
+      configmanagement_sync_repo       = var.configmanagement_sync_repo
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/configmanagement.auto.tfvars"
+}
+
+resource "local_file" "shared_config_initialize_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      initialize_backend_use_case_name    = var.initialize_backend_use_case_name
+      initialize_container_node_pools_cpu = var.initialize_container_node_pools_cpu
+      initialize_container_node_pools_gpu = var.initialize_container_node_pools_gpu
+      initialize_container_node_pools_tpu = var.initialize_container_node_pools_tpu
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/initialize.auto.tfvars"
+}
+
+resource "local_file" "shared_config_networking_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      dynamic_routing_mode = var.dynamic_routing_mode
+      nat_gateway_name     = var.nat_gateway_name
+      network_name         = var.network_name
+      router_name          = var.router_name
+      subnet_cidr_range    = var.subnet_cidr_range
+      subnetwork_name      = var.subnetwork_name
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/networking.auto.tfvars"
+}
+
+resource "local_file" "shared_config_platform_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      platform_name        = var.platform_name
+      resource_name_prefix = var.resource_name_prefix
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/platform.auto.tfvars"
+}
+
+resource "local_file" "shared_config_terraform_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      create_terraform_bucket = var.create_terraform_bucket
+      terraform_project_id    = var.terraform_project_id
+      terraform_write_tfvars  = var.terraform_write_tfvars
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/terraform.auto.tfvars"
+}
+
+resource "local_file" "shared_config_workloads_auto_tfvars" {
+  for_each = toset(var.terraform_write_tfvars ? ["write"] : [])
+
+  content = provider::terraform::encode_tfvars(
+    {
+      #inference_gateway_version = var.inference_gateway_version
+      jobset_version = var.jobset_version
+      kueue_version  = var.kueue_version
+      lws_version    = var.lws_version
+    }
+  )
+  file_permission = "0644"
+  filename        = "${local.shared_config_folder}/workloads.auto.tfvars"
+}
+
 resource "local_file" "use_case_backend_tf" {
   for_each = local.use_case_backend_directories
   content = templatefile(

platforms/gke/base/core/initialize/versions.tf

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 terraform {
-  required_version = ">= 1.5.7"
+  required_version = ">= 1.8.0"
 
   required_providers {
     google = {
@@ -32,6 +32,9 @@ terraform {
       source  = "hashicorp/null"
       version = "3.2.3"
     }
+    terraform = {
+      source = "terraform.io/builtin/terraform"
+    }
   }
 
   provider_meta "google" {

platforms/gke/base/core/teardown.sh

@@ -78,20 +78,18 @@ for terraservice in "${terraservices[@]}"; do
     rm -rf \
       "${ACP_PLATFORM_BASE_DIR}/_shared_config/.terraform/" \
       "${ACP_PLATFORM_BASE_DIR}/_shared_config"/terraform.tfstate* \
+      "${ACP_PLATFORM_BASE_DIR}/kubernetes/kubeconfig" \
+      "${ACP_PLATFORM_BASE_DIR}/kubernetes/manifests" \
       "${ACP_PLATFORM_CORE_DIR}/initialize/.terraform/" \
       "${ACP_PLATFORM_CORE_DIR}/initialize"/terraform.tfstate* \
-      "${ACP_PLATFORM_CORE_DIR}/networking/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/container_cluster/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/container_node_pool/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/gke_enterprise/configmanagement/git/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/gke_enterprise/configmanagement/oci/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/gke_enterprise/fleet_membership/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/gke_enterprise/servicemesh/.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/workloads/kueue.terraform/" \
-      "${ACP_PLATFORM_CORE_DIR}/workloads/kubeconfig" \
-      "${ACP_PLATFORM_CORE_DIR}/workloads/manifests"
+      "${ACP_PLATFORM_CORE_DIR}/workloads/jobset/manifests" \
+      "${ACP_PLATFORM_CORE_DIR}/workloads/kueue/manifests" \
+      "${ACP_PLATFORM_CORE_DIR}/workloads/lws/manifests"
 
     git restore \
+      "${ACP_PLATFORM_BASE_DIR}/_shared_config"/*.auto.tfvars \
+      "${ACP_PLATFORM_BASE_DIR}/kubernetes/kubeconfig/.gitkeep" \
+      "${ACP_PLATFORM_BASE_DIR}/kubernetes/manifests/.gitkeep" \
       "${ACP_PLATFORM_CORE_DIR}/initialize/backend.tf.bucket"
   fi
 done

test/ci-cd/cloudbuild/platforms/gke/base/core/initialize.yaml

@@ -0,0 +1,101 @@
+options:
+  logging: CLOUD_LOGGING_ONLY
+
+steps:
+- id: "Build runner image" 
+  args:
+  - --cache=true
+  - --cache-ttl=48h
+  - --context=dir://test/ci-cd/container_images
+  - --destination=${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest
+  - --dockerfile=test/ci-cd/container_images/dockerfile.runner
+  - --log-format=text
+  - --log-timestamp=false
+  - --verbosity=info
+  name: "gcr.io/kaniko-project/executor:latest"
+  waitFor: ["-"]
+
+- id: "Configure the environment"
+  entrypoint: "test/ci-cd/scripts/terraservice/configure_environment.sh"
+  args:
+  - DEBUG=${_DEBUG}
+  - TF_VAR_cluster_project_id="${PROJECT_ID}-$${PROJECT_SUFFIX}"
+  - TF_VAR_platform_name="${SHORT_SHA}"
+  - TF_VAR_terraform_project_id="${PROJECT_ID}-$${PROJECT_SUFFIX}"
+  env:
+  - BUILD_ID=${BUILD_ID}
+  - DEBUG=${_DEBUG}
+  - PROJECT_ID=${PROJECT_ID}
+  - SHORT_SHA=${SHORT_SHA}
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Build runner image"
+
+- id: "Core Terraservice 'initialize'"
+  entrypoint: "test/ci-cd/scripts/terraservice/apply_initialize.sh"
+  args:
+  - /workspace/platforms/gke/base/core
+  - initialize
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Configure the environment"
+
+- id: "Verify Base shared configurations"
+  entrypoint: "test/ci-cd/scripts/terraservice/verify_configs.sh"
+  args:
+  - /workspace/platforms/gke/base/_shared_config
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Core Terraservice 'initialize'"
+
+- id: "Core Terraservice 'networking'"
+  entrypoint: "test/ci-cd/scripts/terraservice/apply.sh"
+  args:
+  - /workspace/platforms/gke/base/core
+  - networking
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Verify Base shared configurations"
+
+- id: "Core Terraservice 'container_cluster'"
+  entrypoint: "test/ci-cd/scripts/terraservice/apply.sh"
+  args:
+  - /workspace/platforms/gke/base/core
+  - container_cluster
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Core Terraservice 'networking'"
+
+- id: "Core Terraservice 'networking' check for changes"
+  entrypoint: "test/ci-cd/scripts/terraservice/plan.sh"
+  args:
+  - /workspace/platforms/gke/base/core
+  - networking
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Core Terraservice 'container_cluster'"
+
+- id: "Core Terraservice 'container_cluster' check for changes"
+  entrypoint: "test/ci-cd/scripts/terraservice/plan.sh"
+  args:
+  - /workspace/platforms/gke/base/core
+  - container_cluster
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Core Terraservice 'networking' check for changes"
+
+- id: "Delete the environment"
+  entrypoint: "test/ci-cd/scripts/terraservice/delete_environment.sh"
+  args:
+  - DEBUG=${_DEBUG}
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+
+- id: "Set build status"
+  entrypoint: "test/ci-cd/scripts/terraservice/set_build_status.sh"
+  name: "${_AR_REPO_LOCATION}-docker.pkg.dev/${PROJECT_ID}/ci-cd/runner:latest"
+  waitFor:
+  - "Delete the environment"
+
+substitutions:
+  _AR_REPO_LOCATION: "us-central1"
+  _DEBUG: "false"

test/ci-cd/scripts/terraservice/verify_configs.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -o nounset
+set -o pipefail
+
+source /workspace/build.env
+if [ "${DEBUG,,}" == "true" ]; then
+  set -o xtrace
+fi
+
+SHARED_CONFIG_FOLDER="${1}"
+
+exit_handler() {
+  exit_code=$?
+
+  if [ ${exit_code} -ne 0 ]; then
+    echo "Configuration mismatch in '${SHARED_CONFIG_FOLDER}'" >>/workspace/build-failed.lock
+  fi
+
+  exit 0
+}
+trap exit_handler EXIT
+
+errors=0
+configs=$(ls ${SHARED_CONFIG_FOLDER}/*.auto.tfvars | sed 's|.*/||' | sed -e 's|\(.auto.tfvars\)*$||')
+for config in ${configs}; do
+  echo "Checking '${config}'..."
+  variables=$(grep -e '^variable "' ${SHARED_CONFIG_FOLDER}/${config}_variables.tf | sed 's|^[^"]*"\([^"]*\)".*|\1|' | sort)
+  tfvars=$(grep '^[[:alnum:]]' ${SHARED_CONFIG_FOLDER}/${config}.auto.tfvars | sed -E 's/^([a-zA-Z_][a-zA-Z0-9_]*)\s*=.*$/\1/g' | sort)
+  diff <(echo "$variables") <(echo "$tfvars")
+
+  if [ $? == 0 ]; then
+    echo -e "[MATCH]\n"
+  else
+    errors=$((errors + 1))
+    echo -e "[MISMATCH]\n"
+  fi
+done
+
+exit ${errors}