Bump workflow and cluster versions

Author: Jaykul

.github/workflows/deploy.yaml

@@ -19,10 +19,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Login to Azure
-        uses: azure/login@v2.1.1
+        uses: azure/login@v2
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}

Cluster.bicep

@@ -114,9 +114,9 @@ param additionalNodePoolProfiles array = []
 
 @description('''
 Optional. The base version of Kubernetes to use. Node pools are set to auto patch, so they only use the 'major.minor' part.
-Defaults to 1.28
+Defaults to 1.30
 ''')
-param kubernetesVersion string = '1.28'
+param kubernetesVersion string = '1.30'
 
 @description('''Optional. Controls automatic upgrades:
 - none. No automatic patching
@@ -218,7 +218,7 @@ module waitForRole 'modules/deploymentScript.bicep' = {
   params: {
     name: 'waitForRoleAssignment'
     location: location
-    azPowerShellVersion : '11.0'
+    azPowerShellVersion : '13.2'
     userAssignedIdentityResourceID: controlPlaneId.outputs.id
     timeout: 'PT60M'
     scriptContent : join([
@@ -253,8 +253,6 @@ module waitForRole 'modules/deploymentScript.bicep' = {
 // }
 
 
-
-
 module keyVault 'modules/keyVault.bicep' = {
   name: '${deploymentName}_keyvault'
   params: {
@@ -313,10 +311,9 @@ module fluxId 'modules/userAssignedIdentity.bicep' = {
 @description('Optional. If true, skips Flux extension (you can still deploy it later, or by hand).')
 param installFluxManually bool = false
 
-// // Managed Flux
+// Managed Flux (obviously depends on the fluxId which depends on aks)
 module flux 'modules/flux.bicep' = if (!installFluxManually) {
   name: '${deploymentName}_flux'
-  dependsOn: [ aks, fluxId ]
   params: {
     baseName: baseName
     identityClientId: fluxId.outputs.clientId

README.md

@@ -46,9 +46,9 @@ New-AzResourceGroupDeployment @Deployment -OutVariable Results
 
 ## GitOps Configuration
 
-One thing to note is that I am using a _public_ repository ([PoshCode/Cluster](/PoshCode/Cluster)) for my GitOps configuration. Because it's public, there's no need to configure a PAT token or anything for Flux to be able to access it.
+One thing to note is that I am using a _public_ repository ([PoshCode/Cluster](/PoshCode/Cluster)) for my GitOps configuration. Because it's public, there's no need to configure any sort of authentication tokens for Flux to be able to access it.
 
-I'm currently using the Azure Kubernetes Flux Extension to install Flux CD for GitOps. That dramatically simplifies everything, because the Bicep deployment is literally all that's required to deploy the working cluster. However, if you needed to configure the credentials, you would just pass `gitOpsGitUsername` and `gitOpsGitPassword` as part of the `TemplateParameterObject`. There is a feature coming later this year to Flux to support Workflow Identity for git authentication, but for now you need to use a read-only deploy token or something.
+I'm currently using the Azure Kubernetes Flux Extension to install Flux CD for GitOps, this dramatically simplifies configuration for Flux: when the Bicep deployment is complete, Flux is already running on the cluster.
 
 ### Manually bootstrapping Flux
 
@@ -58,31 +58,41 @@ If you wanted to install flux by hand on an existing cluster, it can be as simpl
 flux bootstrap github --owner PoshCode --repository cluster --path=clusters/poshcode
 ```
 
-But if you need to customize workload identity, it can get a lot more complex, because you'll need to patch the flux deployment.
+But if you need to customize workload identity, it can get a bit more complex, but Workload Identity is supported now for access to [Azure DevOps](https://fluxcd.io/flux/components/source/gitrepositories/#azure) and [GitHub](https://fluxcd.io/flux/components/source/gitrepositories/#github), at least.
 
 ## CURRENT STATUS WARNING
 
 I'm playing with Cilium Gateway API, so I've set the network plugin to "none" so that I can take control of the cilium install.
 
+Since the Gateway API in Cilium is part of their Service Mesh, it doesn't seem Azure's AKS team is too keen on getting it working out of the box, so I have to install it manually.
+
+NOTE: in order to use the Gateway API, you need to install the Gateway CRDs. That's handled after the cluster install by Flux.
+However, Cilium CNI has to be installed _before the nodes can even connect_, so it's basically a multipart install, which I have not automated.
+
+1. Install the cluster with the network plugin set to "none"
+2. Install Cilium CNI, and then the nodes will come up.
+3. Install the Gateway API CRDs, and then the Gateway API will be available.
+4. "Upgrade" cilium to enable the gateway API.
+
 Installing the cilium tools is as simple as downloading the right release from their GitHub release pages and unzipping.
 
 ```PowerShell
-Install-GitHubRelease cilium cilium
+Install-GitHubRelease cilium cilium-cli
 Install-GitHubRelease cilium hubble
 ```
 
 And installing it into the AKS cluster is just this, using the same `"rg-$name"` value as the resource group deployment:
 
 ```PowerShell
-cilium install --version 1.15.3 --set azure.resourceGroup="rg-$name" --set kubeProxyReplacement=true --set gatewayAPI.enabled=true
+cilium install --version 1.17.0 --set azure.resourceGroup="rg-$name" --set kubeProxyReplacement=true --set gatewayAPI.enabled=true
 ```
 
 If you want to complete the deployment in a single pass, you have to `Import-AzAksCredential` as soon as the cluster shows up in Azure, and then once `kubectl get nodes` shows all your nodes (they won't come up ready, because they won't have a network), you can run the `cilium install` while Azure is showing the Flux deployment is still running (it won't complete successfully until after cilium is installed, so if you don't run the install, it will fail after the time-out, and you'll have to re-run the deployment).
 
-Currently, I'm running it with prometheus enabled, which is more like:
+Once the cluster is up, and you've installed the Gateway API CRDs, you can run the `cilium upgrade` command to enable the Gateway API. I'm _also_ enabling hubble and prometheus:
 
 ```PowerShell
-cilium upgrade --version 1.15.3 --set azure.resourceGroup="rg-$name" `
+cilium install --version 1.17.0 --set azure.resourceGroup="rg-$name" `
     --set kubeProxyReplacement=true `
     --set gatewayAPI.enabled=true `
     --set hubble.enabled=true `
@@ -92,4 +102,4 @@ cilium upgrade --version 1.15.3 --set azure.resourceGroup="rg-$name" `
     --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}"
 ```
 
-I haven't even tried to automate this, because I'm honestly not sure I'll keep using cilium gateway, and I still hope the AKS team will expose settings for this option.
+Given it's been more than a year, and Azure's "CNI powered by Cilium" still lists L7 policy enforcement as a limmitation, I still have not tried to use that _and_ cilium gateway, so I should probably go ahead and get the Cilium Helm Chart into my GitOps repo 😒

modules/managedCluster.bicep

@@ -198,7 +198,7 @@ param kubeletIdentityId string
 // param logAnalyticsWorkspaceResourceID string
 
 @description('A private AKS Kubernetes cluster')
-resource cluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = {
+resource cluster 'Microsoft.ContainerService/managedClusters@2024-10-01' = {
   name: 'aks-${baseName}'
   location: location
   tags: tags
@@ -404,56 +404,54 @@ resource cluster 'Microsoft.ContainerService/managedClusters@2024-01-01' = {
 
 resource agentPools 'Microsoft.ContainerService/managedClusters/agentPools@2024-01-01' = [for (pool, index) in additionalNodePools: {
   // The default is 6 characters long because that's the max for Windows nodes
-  name: contains(pool, 'name') ? pool.name : format('npuser{0:D2}', index)
+  name: pool.?name ?? format('npuser{0:D2}', index)
   parent: cluster
   properties: {
-    availabilityZones: contains(pool, 'availabilityZones') ? pool.availabilityZones : []
+    availabilityZones: pool.?availabilityZones ?? []
     // capacityReservationGroupID: 'string'
-    count: contains(pool, 'count') ? pool.count : 1
+    count: pool.?count ?? 1
     // creationData: { sourceResourceId: 'string' }
-    enableAutoScaling: contains(pool, 'enableAutoScaling') ? pool.enableAutoScaling : true
+    enableAutoScaling: pool.?enableAutoScaling ?? true
     // enableCustomCATrust: contains(pool, 'enableCustomCATrust') ? pool.enableCustomCATrust : false
-    enableEncryptionAtHost: contains(pool, 'enableEncryptionAtHost') ? pool.enableEncryptionAtHost : false
-    enableFIPS: contains(pool, 'enableFIPS') ? pool.enableFIPS : false
-    enableNodePublicIP: contains(pool, 'enableNodePublicIP') ? pool.enableNodePublicIP : false
-    enableUltraSSD: contains(pool, 'enableUltraSSD') ? pool.enableUltraSSD : false
+    enableEncryptionAtHost: pool.?enableEncryptionAtHost ?? false
+    enableFIPS: pool.?enableFIPS ?? false
+    enableNodePublicIP: pool.?enableNodePublicIP ?? false
+    enableUltraSSD: pool.?enableUltraSSD ?? false
     // gpuInstanceProfile: 'string'
     // hostGroupID: 'string'
     // podSubnetID: 'string'
-    kubeletConfig: contains(pool, 'kubeletConfig') ? pool.kubeletConfig : null // If not null, causes error: CustomKubeletConfig or CustomLinuxOSConfig can not be changed for this operation.
-    kubeletDiskType: contains(pool, 'kubeletDiskType') ? pool.kubeletDiskType : 'OS'
-    linuxOSConfig: contains(pool, 'linuxOSConfig') ? pool.linuxOSConfig : null
-    maxCount: contains(pool, 'maxCount') ? pool.maxCount : 10 // int
-    maxPods: contains(pool, 'maxPods') ? pool.maxPods : maxPodsPerNode // int
-    minCount: contains(pool, 'minCount') ? pool.minCount : 1 // int
-    mode: contains(pool, 'mode') ? pool.mode : 'User' // 'string'
-    networkProfile: contains(pool, 'networkProfile') ? pool.networkProfile : {}
-
-    nodeLabels: contains(pool, 'nodeLabels') ? pool.nodeLabels : {} // {}
+    kubeletConfig: pool.?kubeletConfig ?? null
+    kubeletDiskType: pool.?kubeletDiskType ?? 'OS'
+    linuxOSConfig: pool.?linuxOSConfig ?? null
+    maxCount: pool.?maxCount ?? 10
+    maxPods: pool.?maxPods ?? maxPodsPerNode
+    minCount: pool.?minCount ?? 1
+    mode: pool.?mode ?? 'User'
+    networkProfile: pool.?networkProfile ?? {}
+    nodeLabels: pool.?nodeLabels ?? {}
     // nodePublicIPPrefixID: Doesn't support being empty
-    nodeTaints: contains(pool, 'nodeTaints') ? pool.nodeTaints : [] // ['string' ]
+    nodeTaints: pool.?nodeTaints ?? []
     orchestratorVersion: join(take(split(kubernetesVersion, '.'), 2), '.')
-    osDiskSizeGB: contains(pool, 'osDiskSizeGB') ? pool.osDiskSizeGB : 128 // int
-    osDiskType: contains(pool, 'osDiskType') ? pool.osDiskType : 'Ephemeral' // 'string'
-    osSKU: contains(pool, 'osSKU') ? pool.osSKU : 'Ubuntu' // 'string'
-    osType: contains(pool, 'osType') ? pool.osType : 'Linux' // 'string'
+    osDiskSizeGB: pool.?osDiskSizeGB ?? 128
+    osDiskType: pool.?osDiskType ?? 'Ephemeral'
+    osSKU: pool.?osSKU ?? 'Ubuntu'
+    osType: pool.?osType ?? 'Linux'
     // podSubnetID: 'string'
     // powerState: contains(pool,'powerState') ? pool.powerState : //
     // proximityPlacementGroupID: contains(pool,'proximityPlacementGroupID') ? pool.proximityPlacementGroupID : // 'string'
-    scaleDownMode: contains(pool, 'scaleDownMode') ? pool.scaleDownMode : 'Delete' // 'string'
-    scaleSetEvictionPolicy: contains(pool, 'scaleSetEvictionPolicy') ? pool.scaleSetEvictionPolicy : 'Delete' // 'string'
+    scaleDownMode: pool.?scaleDownMode ?? 'Delete'
+    scaleSetEvictionPolicy: pool.?scaleSetEvictionPolicy ?? 'Delete'
     // scaleSetPriority: contains(pool,'scaleSetPriority') ? pool.scaleSetPriority : 'Regular' // causes error:  Changing property 'properties.ScaleSetPriority' is not allowed.
     // spotMaxPrice: contains(pool,'spotMaxPrice') ? pool.spotMaxPrice : 0
-    type: contains(pool, 'type') ? pool.type : 'VirtualMachineScaleSets'
-    tags: contains(pool, 'tags') ? pool.tags : tags // tags
-    upgradeSettings: contains(pool, 'upgradeSettings') ? pool.upgradeSettings : { maxSurge: '33%' }
-    vmSize: contains(pool, 'vmSize') ? pool.vmSize : 'Standard_DS2_v2' // 'string'
+    type: pool.?type ?? 'VirtualMachineScaleSets'
+    tags: pool.?tags ?? tags
+    upgradeSettings: pool.?upgradeSettings ?? { maxSurge: '33%' }
+    vmSize: pool.?vmSize ?? 'Standard_DS2_v2'
     // vnetSubnetID: contains(pool, 'vnetSubnetID') ? pool.vnetSubnetID : nodeSubnetId // 'string'
     // windowsProfile: contains(pool,'windowsProfile') ? pool.windowsProfile : {} // { }
-    workloadRuntime: contains(pool, 'workloadRuntime') ? pool.workloadRuntime : 'OCIContainer' // 'string'
+    workloadRuntime: pool.?workloadRuntime ?? 'OCIContainer'
   }
 }]
-
 @description('The id of the AKS cluster')
 output id string = cluster.id