diff --git a/.gitignore b/.gitignore index 49f05b1e9d8d..05fcd3cc8787 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,16 @@ api_key generation/new_client/workspace # Monorepo repository generation -monorepo \ No newline at end of file +monorepo + +# Terraform +*.hcl +**/.terraform/.terraform/ +**/.terraform/plugins/ +**/.terraform/providers/ +**/.terraform/plugin_path +*.lock. +*.tfstate +*.tfstate.backup +*.tfstate.*.backup +*.tfstate.lock.info diff --git a/.kokoro/build.sh b/.kokoro/build.sh index ccd472bba28c..b4aeb114d131 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -47,6 +47,13 @@ case ${JOB_TYPE} in IFS=, echo "${modified_module_list[*]}" ) + +# terraform -version +# source ./.terraform/helpers/init.sh "$module_list" +# source ./.terraform/helpers/plan.sh +# source ./.terraform/helpers/apply.sh +# source ./.terraform/helpers/populate-env.sh + install_modules printf "Running Integration Tests for:\n%s\n" "${module_list}" mvn -B ${INTEGRATION_TEST_ARGS} \ @@ -66,6 +73,9 @@ case ${JOB_TYPE} in -T 1C \ verify RETURN_CODE=$? + +# source ./.terraform/helpers/destroy.sh + printf "Finished Integration Tests for:\n%s\n" "${module_list}" else echo "No Integration Tests to run" diff --git a/.terraform/.gitignore b/.terraform/.gitignore new file mode 100644 index 000000000000..a4dfcdbb1d06 --- /dev/null +++ b/.terraform/.gitignore @@ -0,0 +1,7 @@ +generated.tfplan +generated.tfplan.json +generated.auto.tfvars +generated-env.sh +generated-main.tf +generated-outputs.tf +generated-variables.tf diff --git a/.terraform/cleanup.sh b/.terraform/cleanup.sh new file mode 100755 index 000000000000..5cecb1b5c88a --- /dev/null +++ b/.terraform/cleanup.sh @@ -0,0 +1,31 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$scriptDir" >/dev/null + +# Ensure GCP project environment variables are initialized. +if [[ $(terraform state list) == "" ]]; then + echo "Nothing to destroy." + exit +fi + +source ./helpers/gcloud-sync-env.sh +source ./helpers/destroy.sh +source ./helpers/gcloud-delete-project.sh + +popd >/dev/null diff --git a/.terraform/helpers/apply.sh b/.terraform/helpers/apply.sh new file mode 100755 index 000000000000..2a1afe965cd3 --- /dev/null +++ b/.terraform/helpers/apply.sh @@ -0,0 +1,21 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null || exit +terraform apply "generated.tfplan" || exit +popd >/dev/null || exit diff --git a/.terraform/helpers/common.sh b/.terraform/helpers/common.sh new file mode 100644 index 000000000000..c987de476629 --- /dev/null +++ b/.terraform/helpers/common.sh @@ -0,0 +1,71 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Find all directories starting with 'java-', sort them, then join +# with ',' as the delimiter. +function listAllModules() { + # Ensure current directory is repo root. + helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + pushd "$helperDir/../.." >/dev/null + + ls -1 -d java-* | sort | paste -s -d, - + + popd >/dev/null +} + +# Replaces '-' with '_' to get a Terraform output-friendly label +function getFriendlyOutputName() { + echo "$1" | tr '-' _ +} + +# Get the output object in JSON format for the given module. +function getOutput() { + friendlyName=$(getFriendlyOutputName "$1") + terraform output -json "$friendlyName" +} + +# Parse stdin and get the value associated with the given key. +function parseJson() { + python3 -c "import sys, json; print(json.load(sys.stdin)['$1'])" +} + +# Example use: getModuleOutput java-redis redis_network +function getModuleOutput() { + getOutput "$1" | parseJson "$2" +} + +# @returns exit code 0 if list $1 contains entry $2. +function contains() { + echo "$1" | grep -w -q "$2" +} + +# @returns a new-line delimited list of active terraform modules +function getActiveTerraformModules() { + terraform state list | awk -F'[/.]' '{print $2}' | uniq +} + +function getTerraformServiceAccountName() { + echo "terraform-service-account" +} + +function getTerraformServiceAccountEmail() { + if [ -z "${GOOGLE_CLOUD_PROJECT}" ]; then + echo "GOOGLE_CLOUD_PROJECT must be defined." + exit 1 + fi + echo "$(getTerraformServiceAccountName)@$GOOGLE_CLOUD_PROJECT.iam.gserviceaccount.com" +} diff --git a/.terraform/helpers/destroy.sh b/.terraform/helpers/destroy.sh new file mode 100755 index 000000000000..64302409a171 --- /dev/null +++ b/.terraform/helpers/destroy.sh @@ -0,0 +1,39 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +# Execute 'predestroy.sh' scripts for any active modules +source ./helpers/common.sh +allModules=$(listAllModules) +activeModules=$(getActiveTerraformModules) +IFS=',' +for module in $allModules; do + friendlyName=$(getFriendlyOutputName "$module") + if ! contains "$activeModules" "$friendlyName"; then + continue # Skip unless active. + fi + + if [[ -f "../$module/.terraform/predestroy.sh" ]]; then + # shellcheck disable=SC1090 + source "../$module/.terraform/predestroy.sh" + fi +done + +terraform destroy -auto-approve +popd >/dev/null diff --git a/.terraform/helpers/gcloud-create-project.sh b/.terraform/helpers/gcloud-create-project.sh new file mode 100755 index 000000000000..513d65977bf7 --- /dev/null +++ b/.terraform/helpers/gcloud-create-project.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +if [ -n "${GOOGLE_CLOUD_PROJECT}" ]; then + echo "Using current GOOGLE_CLOUD_PROJECT: $GOOGLE_CLOUD_PROJECT" + return +fi + +currentProject=$(gcloud config get project) +if [ -n "${currentProject}" ]; then + echo -n "Do you want to use the current gcloud project ($currentProject)? (Y|n): " + read -r shouldUseCurrent + if [[ "$shouldUseCurrent" != n* ]] && [[ "$shouldUseCurrent" != N* ]]; then + GOOGLE_CLOUD_PROJECT=$currentProject + export GOOGLE_CLOUD_PROJECT + return + fi +fi + +echo -n "GOOGLE_CLOUD_PROJECT not set. Do you want to create a project? (Y|n): " +read -r shouldCreate +if [[ "$shouldCreate" == n* ]] || [[ "$shouldCreate" == N* ]]; then + echo "Project required. Exiting." + exit 1 +fi + +# Ensure required environment variables are set. +if [ -z "${GOOGLE_CLOUD_FOLDER_ID}" ]; then + echo -n "GOOGLE_CLOUD_FOLDER_ID not set. GCP Folder ID: " + read -r folder_id + export GOOGLE_CLOUD_FOLDER_ID="${folder_id}" +fi +if [ -z "${GOOGLE_CLOUD_BILLING_ACCOUNT}" ]; then + echo -n "GOOGLE_CLOUD_BILLING_ACCOUNT not set. GCP Billing Account ID: " + read -r billing_acct + export GOOGLE_CLOUD_BILLING_ACCOUNT="${billing_acct}" +fi +if [ -z "${GOOGLE_CLOUD_PROJECT_PREFIX}" ]; then + echo -n "GOOGLE_CLOUD_PROJECT_PREFIX not set. Prefix for New Project: " + read -r prefix + export GOOGLE_CLOUD_PROJECT_PREFIX="${prefix}" +fi + +# Provision GCP Project +projectId="${GOOGLE_CLOUD_PROJECT_PREFIX}"-"$RANDOM" +gcloud projects create --folder="$GOOGLE_CLOUD_FOLDER_ID" "$projectId" || exit +gcloud config set project "$projectId" +gcloud services enable cloudresourcemanager.googleapis.com +gcloud beta billing projects link "$projectId" --billing-account="$GOOGLE_CLOUD_BILLING_ACCOUNT" +GOOGLE_CLOUD_PROJECT=$projectId diff --git a/.terraform/helpers/gcloud-create-service-account.sh b/.terraform/helpers/gcloud-create-service-account.sh new file mode 100644 index 000000000000..ab72e4c5e27c --- /dev/null +++ b/.terraform/helpers/gcloud-create-service-account.sh @@ -0,0 +1,63 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Use the project ID in gcloud set-quota-project. Clear the existing quota project directly from +# the configuration, and re-set. +gcloud config set project "$GOOGLE_CLOUD_PROJECT" +sed -i.bak '/quota_project_id/d' ~/.config/gcloud/application_default_credentials.json +gcloud auth application-default set-quota-project "$GOOGLE_CLOUD_PROJECT" + +# Assign permission for current gcloud account to impersonate a service account. +gcloud_account=$(gcloud config get account) +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="user:$gcloud_account" \ + --role="roles/iam.serviceAccountTokenCreator" >/dev/null + +# Set up service account for impersonation +source ./helpers/common.sh +service_account_name=$(getTerraformServiceAccountName) +service_account_email=$(getTerraformServiceAccountEmail) +# If it doesn't already exist, create the service account. +gcloud iam service-accounts describe "$service_account_email" &>/dev/null +if [[ $? -ne 0 ]]; then + gcloud iam service-accounts create "$service_account_name" + createdServiceAccount=true +else + createdServiceAccount=false +fi + +# Assign permissions to the service account. +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:$service_account_email" \ + --role="roles/owner" >/dev/null +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:$service_account_email" \ + --role="roles/resourcemanager.projectIamAdmin" >/dev/null + +# See https://cloud.google.com/blog/topics/developers-practitioners/using-google-cloud-service-account-impersonation-your-terraform-code +export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$service_account_email + +if $createdServiceAccount; then + echo "Waiting 2m for service account permissions to take effect... [0s elapsed]" + sleep 30 + echo "Waiting 2m for service account permissions to take effect... [30s elapsed]" + sleep 30 + echo "Waiting 2m for service account permissions to take effect... [1m0s elapsed]" + sleep 30 + echo "Waiting 2m for service account permissions to take effect... [1m30s elapsed]" + sleep 30 +fi diff --git a/.terraform/helpers/gcloud-delete-project.sh b/.terraform/helpers/gcloud-delete-project.sh new file mode 100644 index 000000000000..87f45200a857 --- /dev/null +++ b/.terraform/helpers/gcloud-delete-project.sh @@ -0,0 +1,35 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +if [ -n "${GOOGLE_CLOUD_PROJECT}" ]; then + helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + pushd "$helperDir/.." >/dev/null + + # Always verify whether or not to destroy the project. + echo -n "Delete project ($GOOGLE_CLOUD_PROJECT)? (y/N): " + read -r shouldDestroy + if [[ "$shouldDestroy" == y* ]] || [[ "$shouldDestroy" == Y* ]]; then + # Do not use service account when attempting to delete the project + unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT + gcloud projects delete "$GOOGLE_CLOUD_PROJECT" + gcloud config unset project + unset GOOGLE_CLOUD_PROJECT + rm ./generated.auto.tfvars + fi + + popd >/dev/null +fi diff --git a/.terraform/helpers/gcloud-login.sh b/.terraform/helpers/gcloud-login.sh new file mode 100755 index 000000000000..e790dbd6dc3a --- /dev/null +++ b/.terraform/helpers/gcloud-login.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Perform gcloud auth login if no current credentials are available. +if gcloud auth print-access-token &>/dev/null; then + true +else + if ! gcloud auth login; then + exit + fi +fi +if gcloud auth application-default print-access-token &>/dev/null; then + true +else + if ! gcloud auth application-default login; then + exit + fi +fi diff --git a/.terraform/helpers/gcloud-sync-env.sh b/.terraform/helpers/gcloud-sync-env.sh new file mode 100644 index 000000000000..728a32bd6396 --- /dev/null +++ b/.terraform/helpers/gcloud-sync-env.sh @@ -0,0 +1,46 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +if ! terraform output -json project_id &>/dev/null; then + if ! terraform refresh &>/dev/null; then + echo "No terraform state found." + exit 1 + fi + if ! terraform output -json project_id &>/dev/null; then + echo "No project found in terraform state." + exit 1 + fi +fi + +# Ensure the gcloud project matches Terraform's current state. +terraform_project_id=$(terraform output -raw project_id) +gcloud_project_id=$(gcloud config get project) || "" +if [[ "$terraform_project_id" != "$gcloud_project_id" ]]; then + echo -n "Do you want to make $terraform_project_id your current gcloud project? (Y/n): " + read -r shouldSwitch + if [[ "$shouldSwitch" == n* ]] || [[ "$shouldSwitch" == N* ]]; then + exit + fi + gcloud config set project "$terraform_project_id" +fi + +GOOGLE_CLOUD_PROJECT="$terraform_project_id" +export GOOGLE_CLOUD_PROJECT + +source ./helpers/common.sh +GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(getTerraformServiceAccountEmail) +export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT diff --git a/.terraform/helpers/generate-config.sh b/.terraform/helpers/generate-config.sh new file mode 100755 index 000000000000..54e05e7df5fe --- /dev/null +++ b/.terraform/helpers/generate-config.sh @@ -0,0 +1,68 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +function initializeGeneratedFiles() { + cp ./helpers/generated-main.template.tf generated-main.tf + cp ./helpers/generated-outputs.template.tf generated-outputs.tf + cp ./helpers/generated-variables.template.tf generated-variables.tf +} + +function appendModule() { + friendlyName=$(getFriendlyOutputName "$1") + + # Append the given module to the generated-main.tf configuration to be + # included in the project's resources during 'terraform apply'. + echo "module \"$friendlyName\" { + source = \"./../$1/.terraform\" + inputs = local.data + depends_on = [time_sleep.for_1m_allowBaseCloudApisToFullyEnable] + }" >>generated-main.tf + + # Append the given module to the generated-output.tf file to provide + # all of this module's outputs as an object. + # See https://www.terraform.io/cli/commands/output + echo "output \"$friendlyName\" { + value = module.$friendlyName + sensitive = true + }" >>generated-outputs.tf +} + +function appendAllModules() { + # Either use given module list, or get a list of all modules in the parent directory. + if [ -n "$1" ]; then + modules=$1 + else + modules=$(listAllModules) + fi + IFS=',' + for module in $modules; do + # Only include modules with a .terraform subdirectory in the generated config. + if [ -d "../$module/.terraform" ]; then + appendModule "${module%/}" # Remove possible trailing '/' + fi + done +} + +# Ensure current directory is /.terraform. +generateDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$generateDir/.." >/dev/null + +source ./helpers/common.sh +initializeGeneratedFiles +appendAllModules "$1" + +popd >/dev/null diff --git a/.terraform/helpers/generated-main.template.tf b/.terraform/helpers/generated-main.template.tf new file mode 100644 index 000000000000..14bde28001f3 --- /dev/null +++ b/.terraform/helpers/generated-main.template.tf @@ -0,0 +1,46 @@ +# Auto-generated by generate-config.sh +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +provider "google" { + project = var.project_id + region = var.region + zone = var.zone +} +locals { + # Objects with additional key-value entries may be passed as variables + # to modules needing an object with only a subset of those entries. + # So this 'data' object is a superset of key-value entries that may be + # needed, and we pass it to every module. + data = { + project_id = var.project_id + region = var.region + zone = var.zone + should_enable_apis_on_apply = var.should_enable_apis_on_apply + should_disable_apis_on_destroy = var.should_disable_apis_on_destroy + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = local.data.project_id + enable_apis = local.data.should_enable_apis_on_apply + disable_services_on_destroy = local.data.should_disable_apis_on_destroy + activate_apis = [ + "cloudresourcemanager.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "serviceusage.googleapis.com", + ] +} +resource "time_sleep" "for_1m_allowBaseCloudApisToFullyEnable" { + create_duration = "1m" + depends_on = [module.project-services] + triggers = { + when_project_created = local.data.project_id + } +} diff --git a/.terraform/helpers/generated-outputs.template.tf b/.terraform/helpers/generated-outputs.template.tf new file mode 100644 index 000000000000..d0e7ae9a9f97 --- /dev/null +++ b/.terraform/helpers/generated-outputs.template.tf @@ -0,0 +1,5 @@ +# Auto-generated by generate-config.sh +# See https://www.terraform.io/cli/commands/output +output "project_id" { + value = local.data.project_id +} diff --git a/.terraform/helpers/generated-variables.template.tf b/.terraform/helpers/generated-variables.template.tf new file mode 100644 index 000000000000..bf072f031423 --- /dev/null +++ b/.terraform/helpers/generated-variables.template.tf @@ -0,0 +1,25 @@ +# Auto-generated by generate-config.sh +variable "project_id" { + type = string + description = "GCP Project ID of the project being used" +} +variable "region" { + type = string + description = "GCP region used to deploy resources" + default = "us-central1" # NOTE: Some integration tests have hardcoded this region. +} +variable "should_enable_apis_on_apply" { + type = bool + default = true + description = "If true, required APIs for active client libraries will be automatically enabled on apply." +} +variable "should_disable_apis_on_destroy" { + type = bool + default = false + description = "If true, any APIs enabled by Terraform during apply will be disabled on destroy." +} +variable "zone" { + type = string + description = "GCP zone used to deploy resources. Must be a zone in the chosen region." + default = "us-central1-c" +} diff --git a/.terraform/helpers/init.sh b/.terraform/helpers/init.sh new file mode 100755 index 000000000000..84050309e8f4 --- /dev/null +++ b/.terraform/helpers/init.sh @@ -0,0 +1,27 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Ensure current directory is same as script. +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +export TF_IN_AUTOMATION=true +source ./helpers/generate-config.sh "$1" +terraform fmt >/dev/null +terraform init + +popd >/dev/null diff --git a/.terraform/helpers/plan.sh b/.terraform/helpers/plan.sh new file mode 100755 index 000000000000..2ae8921e41fe --- /dev/null +++ b/.terraform/helpers/plan.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Ensure current directory is same as script. +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +# If Terraform was previously configured to work with a different project, +# then remove the previous Terraform state. +prev_project_id=$(terraform output -raw project_id 2>/dev/null) +if [[ $? -eq 0 ]]; then + if [[ "$prev_project_id" != "$GOOGLE_CLOUD_PROJECT" ]]; then + if [[ -f terraform.tfstate ]]; then + rm terraform.tfstate + fi + fi +fi + +# Create generated.auto.tfvars which will be used as input values to generated-variables.tf +touch generated.auto.tfvars +echo "# Auto-generated by ./.terraform/plan.sh +project_id = \"$GOOGLE_CLOUD_PROJECT\" +" >generated.auto.tfvars +terraform fmt -list=false generated.auto.tfvars + +# See https://www.terraform.io/cli/commands/plan +terraform plan -out generated.tfplan +terraform show -json generated.tfplan >generated.tfplan.json + +popd >/dev/null diff --git a/.terraform/helpers/populate-env.sh b/.terraform/helpers/populate-env.sh new file mode 100644 index 000000000000..7d4396d5a7ff --- /dev/null +++ b/.terraform/helpers/populate-env.sh @@ -0,0 +1,48 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# @returns exit code 0 if list $1 contains entry $2. +function contains() { + echo "$1" | grep -w -q "$2" +} + +function modifyEnvironment() { + # Set module-specific environment variables for upcoming integration test(s) + if [[ -f "../$1/.terraform/env.sh" ]]; then + # shellcheck disable=SC1090 + source "../$1/.terraform/env.sh" + fi +} + +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +# Invoking 'modifyEnvironment' for any active Terraform mdoule +source ./helpers/common.sh +allModules=$(listAllModules) +activeModules=$(getActiveTerraformModules) +IFS=',' +for module in $allModules; do + friendlyName=$(getFriendlyOutputName "$module") + if ! contains "$activeModules" "$friendlyName"; then + continue # Skip unless active. + fi + + modifyEnvironment "$module" +done + +popd >/dev/null diff --git a/.terraform/setup.sh b/.terraform/setup.sh new file mode 100755 index 000000000000..602b2b26d8b0 --- /dev/null +++ b/.terraform/setup.sh @@ -0,0 +1,28 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$scriptDir" >/dev/null + +source ./helpers/init.sh "$1" +source ./helpers/gcloud-login.sh +source ./helpers/gcloud-create-project.sh +source ./helpers/gcloud-create-service-account.sh +source ./helpers/plan.sh +source ./helpers/apply.sh + +popd >/dev/null diff --git a/.terraform/test.sh b/.terraform/test.sh new file mode 100755 index 000000000000..33576e044b6a --- /dev/null +++ b/.terraform/test.sh @@ -0,0 +1,21 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$scriptDir" >/dev/null +time (source ./setup.sh "$1" && source ./verify.sh "$1") +popd >/dev/null diff --git a/.terraform/verify.sh b/.terraform/verify.sh new file mode 100755 index 000000000000..c21b7bf7d68f --- /dev/null +++ b/.terraform/verify.sh @@ -0,0 +1,62 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +function mvnVerify() { + mvn "$@" \ + ${INTEGRATION_TEST_ARGS} \ + -T 1C \ + -B \ + -ntp \ + -Penable-integration-tests \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dcheckstyle.skip=true \ + -fae \ + verify +} +function testSingle() { + pushd "../$1" >/dev/null + mvnVerify + popd >/dev/null +} +function testAll() { + pushd "../" >/dev/null + mvnVerify -pl -:google-cloud-os-login,-:google-cloud-recommender,-:google-cloud-talent + popd >/dev/null +} + +# Ensure current directory is script directory. +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$scriptDir" >/dev/null + +source ./helpers/gcloud-sync-env.sh +source ./helpers/populate-env.sh + +if [ -n "$1" ]; then + # If given a specific module list, only perform integration tests on those. + IFS=',' + for module in $1; do + testSingle "$module" & + done + wait +else + # If not given an argument list, integration test the parent project. + testAll +fi + +popd >/dev/null diff --git a/java-accessapproval/.terraform/main.tf b/java-accessapproval/.terraform/main.tf new file mode 100644 index 000000000000..36d13e3e7afa --- /dev/null +++ b/java-accessapproval/.terraform/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { source = "hashicorp/google" } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["accessapproval.googleapis.com"] +} diff --git a/java-accessapproval/.terraform/variables.tf b/java-accessapproval/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-accessapproval/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-asset/.terraform/main.tf b/java-asset/.terraform/main.tf new file mode 100644 index 000000000000..1827af53df7f --- /dev/null +++ b/java-asset/.terraform/main.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + google = { source = "hashicorp/google" } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = [ + "cloudasset.googleapis.com", + "cloudresourcemanager.googleapis.com", + "pubsub.googleapis.com", + ] +} +resource "time_sleep" "for_90s_allowPubsubToFullyEnable" { + depends_on = [module.project-services] + create_duration = "90s" +} diff --git a/java-asset/.terraform/variables.tf b/java-asset/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-asset/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-bigqueryconnection/.terraform/env.sh b/java-bigqueryconnection/.terraform/env.sh new file mode 100644 index 000000000000..cee32b92a214 --- /dev/null +++ b/java-bigqueryconnection/.terraform/env.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +MY_SQL_DATABASE=$(getModuleOutput java-bigqueryconnection db_name) +echo "Setting environment variable MY_SQL_DATABASE=$MY_SQL_DATABASE" +export MY_SQL_DATABASE + +MY_SQL_INSTANCE=$(getModuleOutput java-bigqueryconnection db_instance) +echo "Setting environment variable MY_SQL_INSTANCE=$MY_SQL_INSTANCE" +export MY_SQL_INSTANCE + +echo "Setting environment variable DB_PWD=" +DB_PWD=$(getModuleOutput java-bigqueryconnection db_password) +export DB_PWD + +echo "Setting environment variable DB_USER=default" +export DB_USER="default" diff --git a/java-bigqueryconnection/.terraform/main.tf b/java-bigqueryconnection/.terraform/main.tf new file mode 100644 index 000000000000..2a2d49d148f6 --- /dev/null +++ b/java-bigqueryconnection/.terraform/main.tf @@ -0,0 +1,39 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} + +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = [ + "bigquery.googleapis.com", + "bigqueryconnection.googleapis.com", + "sqladmin.googleapis.com", + ] +} + +resource "random_id" "id" { + byte_length = 3 +} +locals { + db_name = lower("mysql-db-${random_id.id.hex}") +} + +module "mysql-db" { + source = "GoogleCloudPlatform/sql-db/google//modules/mysql" + version = "12.0.0" + name = local.db_name + database_version = "MYSQL_8_0" + deletion_protection = false + project_id = var.inputs.project_id + zone = var.inputs.zone + region = var.inputs.region + depends_on = [module.project-services] +} diff --git a/java-bigqueryconnection/.terraform/outputs.tf b/java-bigqueryconnection/.terraform/outputs.tf new file mode 100644 index 000000000000..df13828f2ad4 --- /dev/null +++ b/java-bigqueryconnection/.terraform/outputs.tf @@ -0,0 +1,12 @@ +output "db_name" { + value = local.db_name +} + +output "db_instance" { + value = module.mysql-db.instance_name +} + +output "db_password" { + value = module.mysql-db.generated_user_password + sensitive = true +} diff --git a/java-bigqueryconnection/.terraform/variables.tf b/java-bigqueryconnection/.terraform/variables.tf new file mode 100644 index 000000000000..1127e1061d1e --- /dev/null +++ b/java-bigqueryconnection/.terraform/variables.tf @@ -0,0 +1,12 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + + # GCP zone and region in which to deploy the SQL database. + # Zone must be in the chosen region. + zone = string + region = string + }) +} \ No newline at end of file diff --git a/java-compute/.terraform/main.tf b/java-compute/.terraform/main.tf new file mode 100644 index 000000000000..464ddcd5f394 --- /dev/null +++ b/java-compute/.terraform/main.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} + +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["compute.googleapis.com"] +} diff --git a/java-compute/.terraform/variables.tf b/java-compute/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-compute/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-container/.terraform/env.sh b/java-container/.terraform/env.sh new file mode 100644 index 000000000000..5ac8afddf115 --- /dev/null +++ b/java-container/.terraform/env.sh @@ -0,0 +1,22 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +CONTAINER_NETWORK_NAME=$(getModuleOutput java-container container_network_name) +echo "Setting environment variable CONTAINER_NETWORK_NAME=$CONTAINER_NETWORK_NAME" +export CONTAINER_NETWORK_NAME diff --git a/java-container/.terraform/main.tf b/java-container/.terraform/main.tf new file mode 100644 index 000000000000..fd1bfc2b612b --- /dev/null +++ b/java-container/.terraform/main.tf @@ -0,0 +1,32 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = random_id.id.keepers.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = [ + "compute.googleapis.com", + "container.googleapis.com", + ] +} +resource "random_id" "id" { + byte_length = 3 + keepers = { + project_id = var.inputs.project_id + } +} +locals { + container_network_name = lower("java-container-network-${random_id.id.hex}") +} +resource "google_compute_network" "java_container_network" { + name = local.container_network_name + auto_create_subnetworks = true + depends_on = [module.project-services] +} diff --git a/java-container/.terraform/outputs.tf b/java-container/.terraform/outputs.tf new file mode 100644 index 000000000000..cb1d61996177 --- /dev/null +++ b/java-container/.terraform/outputs.tf @@ -0,0 +1,3 @@ +output "container_network_name" { + value = local.container_network_name +} diff --git a/java-container/.terraform/predestroy.sh b/java-container/.terraform/predestroy.sh new file mode 100644 index 000000000000..7facc5b3621f --- /dev/null +++ b/java-container/.terraform/predestroy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +terraform state rm 'module.java_container.google_compute_network.java_container_network' diff --git a/java-container/.terraform/variables.tf b/java-container/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-container/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java b/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java index 86edf8211d18..544bd91f6419 100644 --- a/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java +++ b/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java @@ -57,7 +57,9 @@ public class ITSystemTest { + "/zones/us-central1-a/clusters/" + CLUSTER_NAME; private static final String NODE_POOL_SEL_LINK = SELF_LINK + "/nodePools/" + NODE_POOL_NAME; - private static final String NETWORK = "java-container-network"; + private static final String CONTAINER_NETWORK_ENV_NAME = "CONTAINER_NETWORK_NAME"; + private static final String DEFAULT_NETWORK = "java-container-network"; + private static final String NETWORK = getContainerNetworkName(); private static final int INITIAL_NODE_COUNT = 1; @BeforeClass @@ -180,4 +182,14 @@ public void listNodePoolsTest() { } } } + + private static String getContainerNetworkName() { + String name = + System.getProperty(CONTAINER_NETWORK_ENV_NAME, System.getenv(CONTAINER_NETWORK_ENV_NAME)); + + if (name == null) { + return DEFAULT_NETWORK; + } + return name; + } } diff --git a/java-datacatalog/.terraform/main.tf b/java-datacatalog/.terraform/main.tf new file mode 100644 index 000000000000..1d5714e3a4f1 --- /dev/null +++ b/java-datacatalog/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["datacatalog.googleapis.com"] +} diff --git a/java-datacatalog/.terraform/variables.tf b/java-datacatalog/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-datacatalog/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dataflow/.terraform/main.tf b/java-dataflow/.terraform/main.tf new file mode 100644 index 000000000000..7913346d145c --- /dev/null +++ b/java-dataflow/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["dialogflow.googleapis.com"] +} diff --git a/java-dataflow/.terraform/variables.tf b/java-dataflow/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dataflow/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-datalabeling/.terraform/main.tf b/java-datalabeling/.terraform/main.tf new file mode 100644 index 000000000000..caad9c3e804b --- /dev/null +++ b/java-datalabeling/.terraform/main.tf @@ -0,0 +1,16 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["datalabeling.googleapis.com"] +} + diff --git a/java-datalabeling/.terraform/variables.tf b/java-datalabeling/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-datalabeling/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dataproc/.terraform/main.tf b/java-dataproc/.terraform/main.tf new file mode 100644 index 000000000000..423881cd5d9b --- /dev/null +++ b/java-dataproc/.terraform/main.tf @@ -0,0 +1,26 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = [ + "compute.googleapis.com", + "dataproc.googleapis.com" + ] +} +data "google_compute_default_service_account" "default" { + depends_on = [module.project-services] +} +resource "google_project_iam_member" "dataproc_iam" { + project = var.inputs.project_id + role = "roles/dataproc.worker" + member = "serviceAccount:${data.google_compute_default_service_account.default.email}" +} diff --git a/java-dataproc/.terraform/variables.tf b/java-dataproc/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dataproc/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dialogflow-cx/.terraform/main.tf b/java-dialogflow-cx/.terraform/main.tf new file mode 100644 index 000000000000..7913346d145c --- /dev/null +++ b/java-dialogflow-cx/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["dialogflow.googleapis.com"] +} diff --git a/java-dialogflow-cx/.terraform/variables.tf b/java-dialogflow-cx/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dialogflow-cx/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dialogflow/.terraform/env.sh b/java-dialogflow/.terraform/env.sh new file mode 100644 index 000000000000..c5bc5bf1903d --- /dev/null +++ b/java-dialogflow/.terraform/env.sh @@ -0,0 +1,22 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +DIALOGFLOW_AGENT=$(getModuleOutput java-dialogflow dialogflow_agent) +echo "Setting environment variable DIALOGFLOW_AGENT=$DIALOGFLOW_AGENT" +export DIALOGFLOW_AGENT diff --git a/java-dialogflow/.terraform/main.tf b/java-dialogflow/.terraform/main.tf new file mode 100644 index 000000000000..d2c22bd4eaec --- /dev/null +++ b/java-dialogflow/.terraform/main.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = random_id.id.keepers.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["dialogflow.googleapis.com"] +} +resource "random_id" "id" { + byte_length = 3 + keepers = { + project_id = var.inputs.project_id + } +} +locals { + agent_display_name = lower("google-cloud-java-tests-${random_id.id.hex}") +} +resource "google_dialogflow_agent" "design_time_agent" { + display_name = local.agent_display_name + default_language_code = "en" + time_zone = "America/Los_Angeles" + match_mode = "MATCH_MODE_HYBRID" + depends_on = [module.project-services] +} diff --git a/java-dialogflow/.terraform/outputs.tf b/java-dialogflow/.terraform/outputs.tf new file mode 100644 index 000000000000..55a93effe9f4 --- /dev/null +++ b/java-dialogflow/.terraform/outputs.tf @@ -0,0 +1,3 @@ +output "dialogflow_agent" { + value = local.agent_display_name +} diff --git a/java-dialogflow/.terraform/variables.tf b/java-dialogflow/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dialogflow/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java b/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java index eb0d20ca4f53..d50ef6443fcd 100644 --- a/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java +++ b/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java @@ -77,8 +77,9 @@ public class ITSystemTest { private static final ProjectName PROJECT_NAME = ProjectName.of(PROJECT_ID); private static final ProjectAgentName PROJECT_AGENT_NAME = ProjectAgentName.of(PROJECT_ID); private static final String DEFAULT_DISPLAY_NAME = "google-cloud-java-tests"; - private static final String DISPLAY_NAME = - System.getProperty("dialogflow.agent", DEFAULT_DISPLAY_NAME); + private static final String LEGACY_AGENT_ENV_NAME = "dialogflow.agent"; + private static final String AGENT_ENV_NAME = "DIALOGFLOW_AGENT"; + private static final String DISPLAY_NAME = getDialogFlowAgentDisplayName(); private static final String TIME_ZONE = "America/Los_Angeles"; private static final String DEFAULT_LANGUAGE_CODE = "en"; private static final String ENTITY_NAME = "test-entity-" + ID; @@ -300,4 +301,14 @@ public void getContextTest() { Context actualContext = contextsClient.getContext(request); assertEquals(context.getName(), actualContext.getName()); } + + private static String getDialogFlowAgentDisplayName() { + String name = System.getProperty(AGENT_ENV_NAME, System.getenv(AGENT_ENV_NAME)); + if (name != null) return name; + + name = System.getProperty(LEGACY_AGENT_ENV_NAME, System.getenv(LEGACY_AGENT_ENV_NAME)); + if (name != null) return name; + + return DEFAULT_DISPLAY_NAME; + } } diff --git a/java-dns/.terraform/main.tf b/java-dns/.terraform/main.tf new file mode 100644 index 000000000000..e5d392529572 --- /dev/null +++ b/java-dns/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["dns.googleapis.com"] +} diff --git a/java-dns/.terraform/variables.tf b/java-dns/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dns/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-errorreporting/.terraform/main.tf b/java-errorreporting/.terraform/main.tf new file mode 100644 index 000000000000..f3fec7f53c83 --- /dev/null +++ b/java-errorreporting/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["clouderrorreporting.googleapis.com"] +} diff --git a/java-errorreporting/.terraform/variables.tf b/java-errorreporting/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-errorreporting/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-iam-admin/.terraform/main.tf b/java-iam-admin/.terraform/main.tf new file mode 100644 index 000000000000..b63f0e3f035b --- /dev/null +++ b/java-iam-admin/.terraform/main.tf @@ -0,0 +1,27 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["iam.googleapis.com"] +} + +resource "random_id" "id" { + byte_length = 3 +} +locals { + service_account_id = lower("service-account-id-${random_id.id.hex}") +} +resource "google_service_account" "service_account" { + account_id = local.service_account_id + display_name = "Service Account" + depends_on = [module.project-services] +} diff --git a/java-iam-admin/.terraform/variables.tf b/java-iam-admin/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-iam-admin/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-iot/.terraform/main.tf b/java-iot/.terraform/main.tf new file mode 100644 index 000000000000..84c84e82c5a0 --- /dev/null +++ b/java-iot/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["cloudiot.googleapis.com"] +} diff --git a/java-iot/.terraform/variables.tf b/java-iot/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-iot/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-kms/.terraform/main.tf b/java-kms/.terraform/main.tf new file mode 100644 index 000000000000..88748c49feeb --- /dev/null +++ b/java-kms/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["cloudkms.googleapis.com"] +} diff --git a/java-kms/.terraform/variables.tf b/java-kms/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-kms/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-notebooks/.terraform/main.tf b/java-notebooks/.terraform/main.tf new file mode 100644 index 000000000000..5b8f6a45b8aa --- /dev/null +++ b/java-notebooks/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["notebooks.googleapis.com"] +} diff --git a/java-notebooks/.terraform/variables.tf b/java-notebooks/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-notebooks/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-notification/.terraform/env.sh b/java-notification/.terraform/env.sh new file mode 100644 index 000000000000..8704dc0f680d --- /dev/null +++ b/java-notification/.terraform/env.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +GOOGLE_STORAGE_SERVICE_AGENT=$(getModuleOutput java-notification storage_service_agent) +echo "Setting environment variable GOOGLE_STORAGE_SERVICE_AGENT=$GOOGLE_STORAGE_SERVICE_AGENT" +export GOOGLE_STORAGE_SERVICE_AGENT diff --git a/java-notification/.terraform/main.tf b/java-notification/.terraform/main.tf new file mode 100644 index 000000000000..57231e778ee6 --- /dev/null +++ b/java-notification/.terraform/main.tf @@ -0,0 +1,22 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} + +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["pubsub.googleapis.com"] +} +resource "time_sleep" "for_90s_allowPubsubToFullyEnable" { + depends_on = [module.project-services] + create_duration = "90s" +} +data "google_storage_project_service_account" "gcs_account" { +} diff --git a/java-notification/.terraform/outputs.tf b/java-notification/.terraform/outputs.tf new file mode 100644 index 000000000000..c6e296e76399 --- /dev/null +++ b/java-notification/.terraform/outputs.tf @@ -0,0 +1,3 @@ +output "storage_service_agent" { + value = data.google_storage_project_service_account.gcs_account.email_address +} diff --git a/java-notification/.terraform/variables.tf b/java-notification/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-notification/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java b/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java index 44e166fef9a8..23ae390fdabc 100644 --- a/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java +++ b/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java @@ -38,6 +38,7 @@ import java.security.spec.InvalidKeySpecException; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -56,6 +57,10 @@ public class ITSystemTest { private static Storage storageService; private static final Logger log = Logger.getLogger(ITSystemTest.class.getName()); + private static final String STORAGE_SERVICE_AGENT = + Optional.ofNullable(System.getenv("GOOGLE_STORAGE_SERVICE_AGENT")) + .map(agent -> "serviceAccount:" + agent) + .orElse("allAuthenticatedUsers"); private static final String BUCKET = RemoteStorageHelper.generateBucketName(); private static final String NAME_SUFFIX = UUID.randomUUID().toString(); private static String projectId; @@ -99,7 +104,7 @@ public void testNotifications() { Policy policy = topicAdminClient.getIamPolicy(topic.toString()); Binding binding = - Binding.newBuilder().setRole("roles/owner").addMembers("allAuthenticatedUsers").build(); + Binding.newBuilder().setRole("roles/owner").addMembers(STORAGE_SERVICE_AGENT).build(); Policy newPolicy = topicAdminClient.setIamPolicy( topic.toString(), policy.toBuilder().addBindings(binding).build()); diff --git a/java-os-login/.terraform/main.tf b/java-os-login/.terraform/main.tf new file mode 100644 index 000000000000..0c72498501e1 --- /dev/null +++ b/java-os-login/.terraform/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { source = "hashicorp/google" } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["oslogin.googleapis.com"] +} diff --git a/java-os-login/.terraform/variables.tf b/java-os-login/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-os-login/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-recommender/.terraform/main.tf b/java-recommender/.terraform/main.tf new file mode 100644 index 000000000000..94b7ea768544 --- /dev/null +++ b/java-recommender/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["recommender.googleapis.com"] +} diff --git a/java-recommender/.terraform/variables.tf b/java-recommender/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-recommender/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java b/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java index 3cabcf4c3b50..e327ee5ee64d 100644 --- a/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java +++ b/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java @@ -32,6 +32,7 @@ import java.util.List; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class ITSystemTest { @@ -58,14 +59,16 @@ public void tearDown() { client.close(); } + @Ignore("Recommendations are not available for brand-new GCP projects.") @Test public void listRecommendationsTest() { ListRecommendationsRequest request = ListRecommendationsRequest.newBuilder().setParent(FORMATTED_PARENT).setFilter("").build(); List recommendations = Lists.newArrayList(client.listRecommendations(request).iterateAll()); - assertThat(recommendations.size() > 0).isTrue(); - assertThat(recommendations.contains(null)).isFalse(); + + assertThat(recommendations).isNotEmpty(); + assertThat(recommendations).doesNotContain(null); } @Test(expected = InvalidArgumentException.class) diff --git a/java-redis/.terraform/env.sh b/java-redis/.terraform/env.sh new file mode 100644 index 000000000000..43ded9b22dba --- /dev/null +++ b/java-redis/.terraform/env.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +REDIS_NETWORK=$(getModuleOutput java-redis redis_network) +echo "Setting environment variable REDIS_NETWORK=$REDIS_NETWORK" +export REDIS_NETWORK diff --git a/java-redis/.terraform/main.tf b/java-redis/.terraform/main.tf new file mode 100644 index 000000000000..580da7002f0e --- /dev/null +++ b/java-redis/.terraform/main.tf @@ -0,0 +1,33 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = [ + "compute.googleapis.com", + "redis.googleapis.com" + ] +} + +resource "random_id" "id" { + byte_length = 3 +} +locals { + redis_vpc_id = lower("redis-vpc-${random_id.id.hex}") +} +resource "google_compute_network" "redis_vpc" { + name = local.redis_vpc_id + depends_on = [module.project-services] +} +resource "time_sleep" "for_2m_allowRedisVpcToFullyEnable" { + depends_on = [google_compute_network.redis_vpc] + create_duration = "2m" +} diff --git a/java-redis/.terraform/outputs.tf b/java-redis/.terraform/outputs.tf new file mode 100644 index 000000000000..abec222f7acf --- /dev/null +++ b/java-redis/.terraform/outputs.tf @@ -0,0 +1,3 @@ +output "redis_network" { + value = local.redis_vpc_id +} diff --git a/java-redis/.terraform/predestroy.sh b/java-redis/.terraform/predestroy.sh new file mode 100644 index 000000000000..32f027e5f95c --- /dev/null +++ b/java-redis/.terraform/predestroy.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +terraform state rm 'module.java_redis.google_compute_network.redis_vpc' diff --git a/java-redis/.terraform/variables.tf b/java-redis/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-redis/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java index 3e67d7db1f6c..e5e42c9b5e7c 100644 --- a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java +++ b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.Calendar; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -46,7 +47,8 @@ public class ITSystemTest { private static final Logger LOG = Logger.getLogger(ITSystemTest.class.getName()); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - private static final String NETWORK = System.getProperty("redis.network", "redis-vpc"); + private static final String NETWORK = + Optional.ofNullable(System.getenv("REDIS_NETWORK")).orElse("redis-vpc"); private static final String INSTANCE = "test-instance-" + UUID.randomUUID().toString().substring(0, 8); private static final String LOCATION = "us-central1"; diff --git a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java index 4da1c16662d4..e9a43b71a516 100644 --- a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java +++ b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java @@ -29,6 +29,7 @@ import com.google.protobuf.FieldMask; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -43,7 +44,8 @@ public class ITSystemTest { private static final Logger LOG = Logger.getLogger(ITSystemTest.class.getName()); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - private static final String NETWORK = System.getProperty("redis.network", "redis-vpc"); + private static final String NETWORK = + Optional.ofNullable(System.getenv("REDIS_NETWORK")).orElse("redis-vpc"); private static final String INSTANCE = "test-instance-" + UUID.randomUUID().toString().substring(0, 8); private static final String LOCATION = "us-central1"; diff --git a/java-resourcemanager/.terraform/main.tf b/java-resourcemanager/.terraform/main.tf new file mode 100644 index 000000000000..018ce549d766 --- /dev/null +++ b/java-resourcemanager/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["cloudresourcemanager.googleapis.com"] +} diff --git a/java-resourcemanager/.terraform/variables.tf b/java-resourcemanager/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-resourcemanager/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-scheduler/.terraform/main.tf b/java-scheduler/.terraform/main.tf new file mode 100644 index 000000000000..09248d9e39c3 --- /dev/null +++ b/java-scheduler/.terraform/main.tf @@ -0,0 +1,26 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + time = { + source = "hashicorp/time" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = [ + "pubsub.googleapis.com", + "cloudscheduler.googleapis.com", + "cloudtrace.googleapis.com", + ] +} +resource "time_sleep" "for_1m_allowServicesTimeToFullyEnable" { + depends_on = [module.project-services] + create_duration = "1m" +} diff --git a/java-scheduler/.terraform/variables.tf b/java-scheduler/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-scheduler/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java index 16552ee3a5a7..bb2321ebb57b 100644 --- a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java +++ b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.ResourceExhaustedException; import com.google.cloud.ServiceOptions; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.scheduler.v1.CloudSchedulerClient; @@ -33,6 +34,8 @@ import com.google.pubsub.v1.ProjectTopicName; import java.util.List; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -145,12 +148,36 @@ public void updateJobTest() { @Test public void runJobTest() throws Exception { RunJobRequest jobRequest = RunJobRequest.newBuilder().setName(JOB_NAME).build(); - ApiFuture job = client.runJobCallable().futureCall(jobRequest); - while (true) { - if (job.isDone()) { - assertJobDetails(job.get()); - break; + + // In a new project, the scheduler queue can take a couple of minutes to initialize + // after the first job is created before it is ready to accept tasks. + retryTwiceIfResourceExhausted( + () -> { + ApiFuture job = client.runJobCallable().futureCall(jobRequest); + while (true) { + if (job.isDone()) { + assertJobDetails(job.get()); + break; + } + } + return null; + }); + } + + private static void retryTwiceIfResourceExhausted(Callable callable) throws Exception { + retryIfResourceExhausted(callable, 2); + } + + private static void retryIfResourceExhausted(Callable callable, int retries) + throws Exception { + try { + callable.call(); + } catch (ResourceExhaustedException ex) { + if (retries == 0) { + throw ex; } + TimeUnit.SECONDS.sleep(60); + retryIfResourceExhausted(callable, retries - 1); } } } diff --git a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java index 00276324172f..a97e8132150c 100644 --- a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java +++ b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals; import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.ResourceExhaustedException; import com.google.cloud.ServiceOptions; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.scheduler.v1beta1.CloudSchedulerClient; @@ -32,6 +33,8 @@ import com.google.pubsub.v1.ProjectTopicName; import java.util.List; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -144,12 +147,36 @@ public void updateJobTest() { @Test public void runJobTest() throws Exception { RunJobRequest jobRequest = RunJobRequest.newBuilder().setName(JOB_NAME).build(); - ApiFuture job = client.runJobCallable().futureCall(jobRequest); - while (true) { - if (job.isDone()) { - assertJobDetails(job.get()); - break; + + // In a new project, the scheduler queue can take a couple of minutes to initialize + // after the first job is created before it is ready to accept tasks. + retryTwiceIfResourceExhausted( + () -> { + ApiFuture job = client.runJobCallable().futureCall(jobRequest); + while (true) { + if (job.isDone()) { + assertJobDetails(job.get()); + break; + } + } + return null; + }); + } + + private static void retryTwiceIfResourceExhausted(Callable callable) throws Exception { + retryIfResourceExhausted(callable, 2); + } + + private static void retryIfResourceExhausted(Callable callable, int retries) + throws Exception { + try { + callable.call(); + } catch (ResourceExhaustedException ex) { + if (retries == 0) { + throw ex; } + TimeUnit.SECONDS.sleep(60); + retryIfResourceExhausted(callable, retries - 1); } } } diff --git a/java-secretmanager/.terraform/main.tf b/java-secretmanager/.terraform/main.tf new file mode 100644 index 000000000000..1a2ac3995974 --- /dev/null +++ b/java-secretmanager/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["secretmanager.googleapis.com"] +} diff --git a/java-secretmanager/.terraform/variables.tf b/java-secretmanager/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-secretmanager/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-speech/.terraform/main.tf b/java-speech/.terraform/main.tf new file mode 100644 index 000000000000..fe91275e185a --- /dev/null +++ b/java-speech/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["speech.googleapis.com"] +} diff --git a/java-speech/.terraform/variables.tf b/java-speech/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-speech/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-talent/.terraform/main.tf b/java-talent/.terraform/main.tf new file mode 100644 index 000000000000..2a5a087a4afc --- /dev/null +++ b/java-talent/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["jobs.googleapis.com"] +} diff --git a/java-talent/.terraform/variables.tf b/java-talent/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-talent/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-tasks/.terraform/main.tf b/java-tasks/.terraform/main.tf new file mode 100644 index 000000000000..5ef89085029d --- /dev/null +++ b/java-tasks/.terraform/main.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["cloudtasks.googleapis.com"] +} +resource "time_sleep" "for_90s_allowTasksToFullyEnable" { + depends_on = [module.project-services] + create_duration = "90s" +} \ No newline at end of file diff --git a/java-tasks/.terraform/variables.tf b/java-tasks/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-tasks/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-texttospeech/.terraform/main.tf b/java-texttospeech/.terraform/main.tf new file mode 100644 index 000000000000..86701e7d8ef3 --- /dev/null +++ b/java-texttospeech/.terraform/main.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["texttospeech.googleapis.com"] +} +resource "time_sleep" "for_90s_allowTextToSpeechToFullyEnable" { + depends_on = [module.project-services] + create_duration = "90s" +} + diff --git a/java-texttospeech/.terraform/variables.tf b/java-texttospeech/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-texttospeech/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-trace/.terraform/main.tf b/java-trace/.terraform/main.tf new file mode 100644 index 000000000000..705f97efaadf --- /dev/null +++ b/java-trace/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["cloudtrace.googleapis.com"] +} diff --git a/java-trace/.terraform/variables.tf b/java-trace/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-trace/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-translate/.terraform/main.tf b/java-translate/.terraform/main.tf new file mode 100644 index 000000000000..c2c8a57e0e3b --- /dev/null +++ b/java-translate/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["translate.googleapis.com"] +} diff --git a/java-translate/.terraform/variables.tf b/java-translate/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-translate/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-video-intelligence/.terraform/main.tf b/java-video-intelligence/.terraform/main.tf new file mode 100644 index 000000000000..1cbb4d4d5211 --- /dev/null +++ b/java-video-intelligence/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["videointelligence.googleapis.com"] +} diff --git a/java-video-intelligence/.terraform/variables.tf b/java-video-intelligence/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-video-intelligence/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-vision/.terraform/main.tf b/java-vision/.terraform/main.tf new file mode 100644 index 000000000000..3869ea8422d7 --- /dev/null +++ b/java-vision/.terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +module "project-services" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + + project_id = var.inputs.project_id + enable_apis = var.inputs.should_enable_apis_on_apply + disable_services_on_destroy = var.inputs.should_disable_apis_on_destroy + activate_apis = ["vision.googleapis.com"] +} diff --git a/java-vision/.terraform/variables.tf b/java-vision/.terraform/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-vision/.terraform/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file