Skip to content

Commit 73cd188

Browse files
nawazkhnawazkh
committed
Update Tiltfile with AKS VNet peering and deletion logic
- update aks-as-mgmt scripts with VNet creation and all clusters deletion - internal LB IP can be set using a env variable
1 parent 2239e3b commit 73cd188

File tree

8 files changed

+150
-36
lines changed

8 files changed

+150
-36
lines changed

‎Tiltfile

Lines changed: 107 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@ settings = {
2323
"capi_version": "v1.8.5",
2424
"caaph_version": "v0.2.5",
2525
"cert_manager_version": "v1.16.1",
26-
"kubernetes_version": "v1.28.3",
27-
"aks_kubernetes_version": "v1.28.3",
26+
"kubernetes_version": "v1.28.15",
27+
"aks_kubernetes_version": "v1.28.15",
2828
"flatcar_version": "3374.2.1",
2929
"azure_location": "eastus",
3030
"control_plane_machine_count": "1",
@@ -212,10 +212,10 @@ def capz():
212212
yaml = str(kustomizesub("./hack/observability")) # build an observable kind deployment by default
213213

214214
# add extra_args if they are defined
215-
if settings.get("extra_args"):
216-
azure_extra_args = settings.get("extra_args").get("azure")
215+
if settings.get("container_args"):
216+
capz_container_args = settings.get("container_args").get("capz-controller-manager")
217217
yaml_dict = decode_yaml_stream(yaml)
218-
append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", azure_extra_args)
218+
append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", capz_container_args)
219219
yaml = str(encode_yaml_stream(yaml_dict))
220220
yaml = fixup_yaml_empty_arrays(yaml)
221221

@@ -317,9 +317,14 @@ def flavors():
317317
for template in template_list:
318318
deploy_worker_templates(template, substitutions)
319319

320+
delete_all_workload_clusters = kubectl_cmd + " delete clusters --all --wait=false;"
321+
322+
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
323+
delete_all_workload_clusters += clear_aks_vnet_peerings()
324+
320325
local_resource(
321326
name = "delete-all-workload-clusters",
322-
cmd = kubectl_cmd + " delete clusters --all --wait=false",
327+
cmd = ["sh", "-ec", delete_all_workload_clusters],
323328
auto_init = False,
324329
trigger_mode = TRIGGER_MODE_MANUAL,
325330
labels = ["flavors"],
@@ -382,16 +387,35 @@ def deploy_worker_templates(template, substitutions):
382387

383388
yaml = shlex.quote(yaml)
384389
flavor_name = os.path.basename(flavor)
385-
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\""
390+
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); "
391+
392+
apiserver_lb_private_ip = os.getenv("AZURE_INTERNAL_LB_PRIVATE_IP", "")
393+
if "windows-apiserver-ilb" in flavor and apiserver_lb_private_ip == "":
394+
flavor_cmd += "export AZURE_INTERNAL_LB_PRIVATE_IP=\"40.0.11.100\"; "
395+
elif "apiserver-ilb" in flavor and apiserver_lb_private_ip == "":
396+
flavor_cmd += "export AZURE_INTERNAL_LB_PRIVATE_IP=\"30.0.11.100\"; "
397+
398+
flavor_cmd += "export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; "
399+
flavor_cmd += "echo \"Cluster ${CLUSTER_NAME} created, don't forget to delete\"; "
386400

387401
# wait for kubeconfig to be available
388-
flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; chmod 600 ./${CLUSTER_NAME}.kubeconfig; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done"
402+
flavor_cmd += "echo \"Waiting for kubeconfig to be available\"; "
403+
flavor_cmd += "until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; "
404+
flavor_cmd += kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; "
405+
flavor_cmd += "chmod 600 ./${CLUSTER_NAME}.kubeconfig; "
406+
flavor_cmd += "echo \"Kubeconfig for ${CLUSTER_NAME} created and saved in the local\"; "
407+
flavor_cmd += "echo \"Waiting for ${CLUSTER_NAME} API Server to be accessible\"; "
408+
flavor_cmd += "until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done; "
409+
flavor_cmd += "echo \"API Server of ${CLUSTER_NAME} is accessible\"; "
389410

390411
# copy the kubeadm configmap to the calico-system namespace.
391412
# This is a workaround needed for the calico-node-windows daemonset to be able to run in the calico-system namespace.
392413
if "windows" in flavor_name:
393-
flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done"
394-
flavor_cmd += "; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -"
414+
flavor_cmd += "until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done; "
415+
flavor_cmd += kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; "
416+
417+
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
418+
flavor_cmd += peer_vnets()
395419

396420
flavor_cmd += get_addons(flavor_name)
397421

@@ -409,14 +433,15 @@ def get_addons(flavor_name):
409433
if "aks" in flavor_name:
410434
return ""
411435

412-
addon_cmd = "; export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}')"
413-
addon_cmd += "; export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ',')"
414-
addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}"
436+
addon_cmd = "export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}'); "
437+
addon_cmd += "export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ','); "
438+
addon_cmd += helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}"
415439
if "flatcar" in flavor_name: # append caCetDir location to the cloud-provider-azure helm install command for flatcar flavor
416440
addon_cmd += " --set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates"
441+
addon_cmd += "; "
417442

418443
if "azure-cni-v1" in flavor_name:
419-
addon_cmd += "; " + kubectl_cmd + " apply -f ./templates/addons/azure-cni-v1.yaml --kubeconfig ./${CLUSTER_NAME}.kubeconfig"
444+
addon_cmd += kubectl_cmd + " apply -f ./templates/addons/azure-cni-v1.yaml --kubeconfig ./${CLUSTER_NAME}.kubeconfig; "
420445
else:
421446
# install calico
422447
if "ipv6" in flavor_name:
@@ -425,7 +450,7 @@ def get_addons(flavor_name):
425450
calico_values = "./templates/addons/calico-dual-stack/values.yaml"
426451
else:
427452
calico_values = "./templates/addons/calico/values.yaml"
428-
addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://docs.tigera.io/calico/charts --version ${CALICO_VERSION} calico tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace"
453+
addon_cmd += helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://docs.tigera.io/calico/charts --version ${CALICO_VERSION} calico tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace; "
429454

430455
return addon_cmd
431456

@@ -454,6 +479,73 @@ def waitforsystem():
454479
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system")
455480
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system")
456481

482+
def peer_vnets():
483+
# TODO: check for az cli to be installed in local
484+
# wait for AKS VNet to be in the state created
485+
peering_cmd = '''
486+
echo \"--------Peering VNETs--------\";
487+
az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180;
488+
export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv);
489+
echo \" 1/8 ${AKS_MGMT_VNET_NAME} found \"; '''
490+
491+
# wait for workload VNet to be created
492+
peering_cmd += '''
493+
az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180;
494+
export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv);
495+
echo \" 2/8 ${CLUSTER_NAME}-vnet found \"; '''
496+
497+
# peer mgmt vnet
498+
peering_cmd += '''
499+
az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none;
500+
az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none;
501+
echo \" 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}\"; '''
502+
503+
# peer workload vnet
504+
peering_cmd += '''
505+
az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none;
506+
az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none;
507+
echo \" 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet\"; '''
508+
509+
# create private DNS zone
510+
peering_cmd += '''
511+
az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none;
512+
az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none;
513+
echo \" 5/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}\"; '''
514+
515+
# link private DNS Zone to workload vnet
516+
peering_cmd += '''
517+
az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none;
518+
az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none;
519+
echo \" 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone\"; '''
520+
521+
# link private DNS Zone to mgmt vnet
522+
peering_cmd += '''
523+
az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none;
524+
az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none;
525+
echo \" 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone\"; '''
526+
527+
# create private DNS zone record
528+
peering_cmd += '''
529+
az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --record-set-name \"@\" --ipv4-address ${AZURE_INTERNAL_LB_PRIVATE_IP} --only-show-errors --output none;
530+
echo \" 8/8 \"@\" private DNS zone record created to point ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com to ${AZURE_INTERNAL_LB_PRIVATE_IP}\"; '''
531+
532+
return peering_cmd
533+
534+
def clear_aks_vnet_peerings():
535+
#
536+
delete_peering_cmd = '''
537+
echo \"--------Clearing AKS MGMT VNETs Peerings--------\";
538+
az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180;
539+
echo \" VNet ${AKS_MGMT_VNET_NAME} found \"; '''
540+
541+
# List all peering names and store them in an array
542+
delete_peering_cmd += '''
543+
PEERING_NAMES=$(az network vnet peering list --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --query \"[].name\" --output tsv);
544+
for PEERING_NAME in ${PEERING_NAMES[@]}; do echo \"Deleting peering: ${PEERING_NAME}\"; az network vnet peering delete --name ${PEERING_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME}; done;
545+
echo \"All VNETs Peerings deleted in ${AKS_MGMT_VNET_NAME}\"; '''
546+
547+
return delete_peering_cmd
548+
457549
##############################
458550
# Actual work happens here
459551
##############################

0 commit comments

Comments
 (0)