aks-as-mgmt.sh

#!/usr/bin/env bash
# Copyright 2024 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit # exit immediately if a command exits with a non-zero status.
set -o nounset # exit when script tries to use undeclared variables.
set -o pipefail # make the pipeline fail if any command in it fails.

REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
# shellcheck source=hack/ensure-azcli.sh
source "${REPO_ROOT}/hack/ensure-azcli.sh" # install az cli and login using WI
# shellcheck source=hack/ensure-tags.sh
source "${REPO_ROOT}/hack/ensure-tags.sh" # set the right timestamp and job name

KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
KIND="${REPO_ROOT}/hack/tools/bin/kind"
AZWI="${REPO_ROOT}/hack/tools/bin/azwi"
make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" "${AZWI##*/}"

export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # management cluster name
export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # resource group name
export AKS_NODE_RESOURCE_GROUP="node-${AKS_RESOURCE_GROUP}"
export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.30.2}"
export AZURE_LOCATION="${AZURE_LOCATION:-westus2}"
export AKS_NODE_VM_SIZE="${AKS_NODE_VM_SIZE:-"Standard_B2s"}"
export AKS_NODE_COUNT="${AKS_NODE_COUNT:-1}"
export MGMT_CLUSTER_KUBECONFIG="${MGMT_CLUSTER_KUBECONFIG:-$REPO_ROOT/aks-mgmt.config}"
export AZURE_IDENTITY_ID_FILEPATH="${AZURE_IDENTITY_ID_FILEPATH:-$REPO_ROOT/azure_identity_id}"
export AZWI_STORAGE_ACCOUNT="capzcioidcissuer${RANDOM_SUFFIX}"
export AZWI_STORAGE_CONTAINER="\$web"
export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:-}"
export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:-}"
export REGISTRY="${REGISTRY:-}"

export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}"
export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}"
export AZURE_TENANT_ID="${AZURE_TENANT_ID:-}"

main() {

  echo "--------------------------------"
  echo "MGMT_CLUSTER_NAME:                    $MGMT_CLUSTER_NAME"
  echo "AKS_RESOURCE_GROUP:                   $AKS_RESOURCE_GROUP"
  echo "AKS_NODE_RESOURCE_GROUP:              $AKS_NODE_RESOURCE_GROUP"
  echo "KUBERNETES_VERSION:                   $KUBERNETES_VERSION"
  echo "AZURE_LOCATION:                       $AZURE_LOCATION"
  echo "AKS_NODE_VM_SIZE:                     $AKS_NODE_VM_SIZE"
  echo "AZURE_NODE_MACHINE_TYPE:              $AZURE_NODE_MACHINE_TYPE"
  echo "AKS_NODE_COUNT:                       $AKS_NODE_COUNT"
  echo "MGMT_CLUSTER_KUBECONFIG:              $MGMT_CLUSTER_KUBECONFIG"
  echo "AZURE_IDENTITY_ID_FILEPATH:           $AZURE_IDENTITY_ID_FILEPATH"
  echo "AZWI_STORAGE_ACCOUNT:                 $AZWI_STORAGE_ACCOUNT"
  echo "AZWI_STORAGE_CONTAINER:               $AZWI_STORAGE_CONTAINER"
  echo "SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH: $SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH"
  echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH"
  echo "REGISTRY:                             $REGISTRY"

  echo "AZURE_SUBSCRIPTION_ID:                $AZURE_SUBSCRIPTION_ID"
  echo "AZURE_CLIENT_ID:                      $AZURE_CLIENT_ID"
  echo "AZURE_TENANT_ID:                      $AZURE_TENANT_ID"
  echo "--------------------------------"

  create_aks_cluster
  set_env_varaibles
}

create_aks_cluster() {
  resource_group_exists=$(az group exists --name "${AKS_RESOURCE_GROUP}" --output tsv)
  if [ "${resource_group_exists}" == 'true' ]; then
    echo "resource group \"${AKS_RESOURCE_GROUP}\" already exists, moving on"
  else
    echo "creating resource group ${AKS_RESOURCE_GROUP}"
    az group create --name "${AKS_RESOURCE_GROUP}" \
    --location "${AZURE_LOCATION}" \
    --output none --only-show-errors \
    --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}"
  fi

  aks_exists=$(az aks show --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" 2>&1 || true) # true because we want to continue if the command fails
  if echo "$aks_exists" | grep -E -q "Resource(NotFound|GroupNotFound)"; then
    echo "creating aks cluster ${MGMT_CLUSTER_NAME} in the resource group ${AKS_RESOURCE_GROUP}"
    az aks create --name "${MGMT_CLUSTER_NAME}" \
    --resource-group "${AKS_RESOURCE_GROUP}" \
    --location "${AZURE_LOCATION}" \
    --kubernetes-version "${KUBERNETES_VERSION}" \
    --node-count "${AKS_NODE_COUNT}" \
    --node-vm-size "${AKS_NODE_VM_SIZE}" \
    --node-resource-group "${AKS_NODE_RESOURCE_GROUP}" \
    --vm-set-type VirtualMachineScaleSets \
    --generate-ssh-keys \
    --network-plugin azure \
    --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" \
    --output none --only-show-errors;
  elif echo "$aks_exists" | grep -q "${MGMT_CLUSTER_NAME}"; then
    echo "cluster ${MGMT_CLUSTER_NAME} already exists in RG ${AKS_RESOURCE_GROUP}, moving on"
  else
    echo "error : ${aks_exists}"
    exit 1
  fi

  # check and save kubeconfig
  echo "saving credentials of cluster ${MGMT_CLUSTER_NAME} in ${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}"
  az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \
  --file "${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" --only-show-errors

  az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \
  --overwrite-existing --only-show-errors

  # echo "fetching Client ID for ${MGMT_CLUSTER_NAME}"
  AKS_MI_CLIENT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
  --only-show-errors | jq -r '.identityProfile.kubeletidentity.clientId')
  export AKS_MI_CLIENT_ID
  echo "mgmt client identity: ${AKS_MI_CLIENT_ID}"
  echo "${AKS_MI_CLIENT_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"

  # echo "fetching Object ID for ${MGMT_CLUSTER_NAME}"
  AKS_MI_OBJECT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
  --only-show-errors | jq -r '.identityProfile.kubeletidentity.objectId')
  export AKS_MI_OBJECT_ID
  echo "mgmt object identity: ${AKS_MI_OBJECT_ID}"

  # echo "fetching Resource ID for ${MGMT_CLUSTER_NAME}"
  AKS_MI_RESOURCE_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
  --only-show-errors | jq -r '.identityProfile.kubeletidentity.resourceId')
  export AKS_MI_RESOURCE_ID
  echo "mgmt resource identity: ${AKS_MI_RESOURCE_ID}"

  # save resource identity name and resource group
  MANAGED_IDENTITY_NAME=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" --output json | jq -r '.name')
  # export MANAGED_IDENTITY_NAME
  echo "mgmt resource identity name: ${MANAGED_IDENTITY_NAME}"
  USER_IDENTITY=$MANAGED_IDENTITY_NAME
  export USER_IDENTITY

  MANAGED_IDENTITY_RG=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" --output json | jq -r '.resourceGroup')
  export MANAGED_IDENTITY_RG
  echo "mgmt resource identity resource group: ${MANAGED_IDENTITY_RG}"

  echo "assigning contributor role to the service principal"
  until az role assignment create --assignee-object-id "${AKS_MI_OBJECT_ID}" --role "Contributor" \
  --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none \
  --only-show-errors; do
    echo "retrying to assign role to the service principal"
    sleep 5
  done

  echo "using ASO_CREDENTIAL_SECRET_MODE as podidentity"
  ASO_CREDENTIAL_SECRET_MODE="podidentity"
}

set_env_varaibles(){
  cat <<EOF > tilt-settings-temp.yaml
kustomize_substitutions:
  MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}"
  AKS_RESOURCE_GROUP: "${AKS_RESOURCE_GROUP}"
  AKS_NODE_RESOURCE_GROUP: "${AKS_NODE_RESOURCE_GROUP}"
  MGMT_CLUSTER_KUBECONFIG: "${MGMT_CLUSTER_KUBECONFIG}"
  AKS_MI_CLIENT_ID: "${AKS_MI_CLIENT_ID}"
  AKS_MI_OBJECT_ID: "${AKS_MI_OBJECT_ID}"
  AKS_MI_RESOURCE_ID: "${AKS_MI_RESOURCE_ID}"
  MANAGED_IDENTITY_NAME: "${MANAGED_IDENTITY_NAME}"
  MANAGED_IDENTITY_RG: "${MANAGED_IDENTITY_RG}"
  AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY: "${AKS_MI_CLIENT_ID}"
  CI_RG: "${MANAGED_IDENTITY_RG}"
  USER_IDENTITY: "${MANAGED_IDENTITY_NAME}"
  CLUSTER_IDENTITY_TYPE: "UserAssignedMSI"
  ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}"
  REGISTRY: "${REGISTRY}"
allowed_contexts:
  - "$MGMT_CLUSTER_NAME"
  - "kind-capz"
azure_location: "${AZURE_LOCATION}"
EOF

# create tilt-settings.yaml if it does not exist
if [ -f tilt-settings.yaml ]; then
  echo "tilt-settings.yaml exists"
else
  echo "tilt-settings.yaml does not exist, creating one"
  touch tilt-settings.yaml
fi

# copy over the existing allowed_contexts to tilt-settings.yaml if it does not exist
allowed_contexts_exists=$(yq eval '.allowed_contexts' tilt-settings.yaml)
if [ "$allowed_contexts_exists" == "null" ]; then
  yq eval '.allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml
fi

# extract allowed_contexts from tilt-settings.yaml
current_contexts=$(yq eval '.allowed_contexts' tilt-settings.yaml | sort -u)

# extract allowed_contexts from tilt-settings-new.yaml
new_contexts=$(yq eval '.allowed_contexts' tilt-settings-temp.yaml | sort -u)

# combine current and new contexts, keeping the union of both
combined_contexts=$(echo "$current_contexts"$'\n'"$new_contexts" | sort -u)

# create a temporary file since env($combined_contexts) is not supported in yq
echo "$combined_contexts" > combined_contexts.yaml

# update allowed_contexts in tilt-settings.yaml with the combined contexts
yq eval --inplace ".allowed_contexts = load(\"combined_contexts.yaml\")" tilt-settings.yaml

# merge the updated kustomize_substitution and azure_location with the existing one in tilt-settings.yaml
yq eval-all 'select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml

mv tilt-settings-new.yaml tilt-settings.yaml
rm -r combined_contexts.yaml
rm -f tilt-settings-temp.yaml
}

main