double confirm with cilium install, cilium connectivity test and sonobuoy run --mode=certified-conformance

    helm template cilium cilium/cilium \
        --version 1.11.4 \
        --namespace kube-system \
        --set bpf.preallocateMaps=false \
        --set cluster.id=0 \
        --set cluster.name=4e8b0505-4c52-57ab-a7f4-481e7ed3a2e3 \
        --set cni.binPath=/usr/libexec/cni \
        --set cni.chainingMode=none \
        --set cni.exclusive=true \
        --set externalIPs.enabled=true \
        --set hostPort.enabled=true \
        --set hostServices.enabled=true \
        --set hubble.enabled=false \
        --set ipam.mode=cluster-pool \
        --set ipam.operator.clusterPoolIPv4MaskSize=24 \
        --set ipam.operator.clusterPoolIPv4PodCIDRList=10.233.64.0/18 \
        --set ipv4.enabled=true \
        --set ipv6.enabled=false \
        --set kubeProxyReplacement=probe \
        --set nodePort.enabled=true \
        --set nodeinit.enabled=true \
        --set operator.replicas=1 \
        --set tunnel=vxlan \
        | yq . > templates/etc/kubernetes/addons/60-cilium.yml.j2

Signed-off-by: Wong Hoi Sing Edison <[email protected]>

Author: hswong3i

defaults/main.yml

@@ -15,12 +15,19 @@
 # limitations under the License.
 
 # Enable native IP masquerade support in eBPF.
-kube_cilium_bpf_masquerade: false
+kube_cilium_bpf_masquerade: ~
 
 # Enables pre-allocation of eBPF map values. This increases memory usage
 # but can reduce latency.
 kube_cilium_bpf_preallocatemaps: false
 
+# Unique ID of the cluster. Must be unique across all connected clusters
+# and in the range of 1 to 255. Only required for Cluster Mesh.
+kube_cilium_cluster_id: "0"
+
+# Name of the cluster. Only required for Cluster Mesh.
+kube_cilium_cluster_name: "{{ hostvars[groups['kube_master'][0]].ansible_machine_id | to_uuid }}"
+
 # Configure the path to the CNI binary directory on the host.
 kube_cilium_cni_binpath: "/usr/libexec/cni"
 
@@ -35,10 +42,10 @@ kube_cilium_cni_exclusive: true
 
 # Enable use of per endpoint routes instead of routing via the
 # cilium_host interface.
-kube_cilium_endpointroutes_enabled: false
+kube_cilium_endpointroutes_enabled: ~
 
 # Enable ExternalIPs service support.
-kube_cilium_externalIPs_enabled: true
+kube_cilium_externalips_enabled: true
 
 # Enable hostPort service support.
 kube_cilium_hostport_enabled: true
@@ -50,7 +57,7 @@ kube_cilium_hostservices_enabled: true
 kube_cilium_hubble_enabled: false
 
 # Configure IP Address Management mode.
-kube_cilium_ipam_mode: "kubernetes"
+kube_cilium_ipam_mode: "cluster-pool"
 
 # IPv4 CIDR mask size to delegate to individual nodes for IPAM.
 kube_cilium_ipam_operator_clusterpoolipv4masksize: "24"
@@ -69,7 +76,7 @@ kube_cilium_ipv6_enabled: false
 
 # Enables waiting for Kubernetes to provide the PodCIDR range via the
 # Kubernetes node resource.
-kube_cilium_k8s_requireipv4podcidr: true
+kube_cilium_k8s_requireipv4podcidr: ~
 
 # Controls how to enable kube-proxy replacement features in BPF datapath.
 kube_cilium_kubeproxyreplacement: "probe"

templates/etc/kubernetes/addons/60-cilium.yml.j2

@@ -124,10 +124,10 @@ data:
   # container image names
   sidecar-istio-proxy-image: "cilium/istio_proxy"
   # Name of the cluster. Only relevant when building a mesh of clusters.
-  cluster-name: default
+  cluster-name: "{{ kube_cilium_cluster_name }}"
   # Unique ID of the cluster. Must be unique across all conneted clusters and
   # in the range of 1 and 255. Only relevant when building a mesh of clusters.
-  cluster-id: ""
+  cluster-id: "{{ kube_cilium_cluster_id }}"
   # Encapsulation mode for communication between nodes
   # Possible values:
   #   - disabled
@@ -146,33 +146,56 @@ data:
   cni-chaining-mode: "{{ kube_cilium_cni_chainingmode }}"
   enable-ipv4-masquerade: "true"
   enable-ipv6-masquerade: "true"
+{% if kube_cilium_bpf_masquerade %}
   enable-bpf-masquerade: "{{ kube_cilium_bpf_masquerade | bool | lower }}"
+{% endif %}
   enable-xt-socket-fallback: "true"
   install-iptables-rules: "true"
   install-no-conntrack-iptables-rules: "false"
   auto-direct-node-routes: "false"
   enable-bandwidth-manager: "false"
   enable-local-redirect-policy: "false"
+{% if kube_cilium_tunnel == "disabled" %}
   ipv4-native-routing-cidr: "{{ kube_cilium_ipv4nativeroutingcidr }}"
+{% endif %}
   kube-proxy-replacement: "{{ kube_cilium_kubeproxyreplacement }}"
+{% if kube_cilium_kubeproxyreplacement != "disabled" %}
   kube-proxy-replacement-healthz-bind-address: ""
+{% endif %}
+{% if kube_cilium_hostservices_enabled %}
   enable-host-reachable-services: "{{ kube_cilium_hostservices_enabled | bool | lower }}"
+{% endif %}
+{% if kube_cilium_hostport_enabled %}
+  enable-host-port: "{{ kube_cilium_hostport_enabled | bool | lower }}"
+{% endif %}
+{% if kube_cilium_externalips_enabled %}
+  enable-external-ips: "{{ kube_cilium_externalips_enabled | bool | lower }}"
+{% endif %}
+{% if kube_cilium_nodeport_enabled %}
+  enable-node-port: "{{ kube_cilium_nodeport_enabled | bool | lower }}"
+{% endif %}
   enable-health-check-nodeport: "true"
   node-port-bind-protection: "true"
   enable-auto-protect-node-port-range: "true"
   enable-session-affinity: "true"
   enable-l2-neigh-discovery: "true"
   arping-refresh-period: "30s"
+{% if kube_cilium_k8s_requireipv4podcidr %}
   k8s-require-ipv4-pod-cidr: "{{ kube_cilium_k8s_requireipv4podcidr | bool | lower }}"
+{% endif %}
+{% if kube_cilium_endpointroutes_enabled %}
   enable-endpoint-routes: "{{ kube_cilium_endpointroutes_enabled | bool | lower }}"
+{% endif %}
   enable-endpoint-health-checking: "true"
   enable-health-checking: "true"
   enable-well-known-identities: "false"
   enable-remote-node-identity: "true"
   operator-api-serve-addr: "127.0.0.1:9234"
   ipam: "{{ kube_cilium_ipam_mode }}"
+{% if kube_cilium_ipam_mode == "cluster-pool" %}
   cluster-pool-ipv4-cidr: "{{ kube_cilium_ipam_operator_clusterpoolipv4podcidrlist }}"
   cluster-pool-ipv4-mask-size: "{{ kube_cilium_ipam_operator_clusterpoolipv4masksize }}"
+{% endif %}
   disable-cnp-status-updates: "true"
   cgroup-root: "/run/cilium/cgroupv2"
   enable-k8s-terminating-endpoint: "true"
@@ -429,13 +452,14 @@ metadata:
   name: cilium
   namespace: kube-system
   labels:
+    k8s-app: cilium
     app.kubernetes.io/name: cilium
     app.kubernetes.io/part-of: kube-system
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/name: cilium
+      k8s-app: cilium
   updateStrategy:
     rollingUpdate:
       maxUnavailable: 2
@@ -449,7 +473,7 @@ spec:
         # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
         scheduler.alpha.kubernetes.io/critical-pod: ""
       labels:
-        app.kubernetes.io/name: cilium
+        k8s-app: cilium
     spec:
       affinity:
         nodeAffinity:
@@ -469,7 +493,7 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             - labelSelector:
                 matchExpressions:
-                  - key: app.kubernetes.io/name
+                  - key: k8s-app
                     operator: In
                     values:
                       - cilium
@@ -611,6 +635,7 @@ spec:
               mountPath: /hostbin
           securityContext:
             privileged: true
+{% if kube_cilium_nodeinit_enabled == true %}
         - name: wait-for-node-init
           image: quay.io/cilium/cilium:v1.11.4
           imagePullPolicy: IfNotPresent
@@ -625,6 +650,7 @@ spec:
           volumeMounts:
             - name: cilium-bootstrap-file-dir
               mountPath: "/tmp/cilium-bootstrap.d"
+{% endif %}
         - name: clean-cilium-state
           image: quay.io/cilium/cilium:v1.11.4
           imagePullPolicy: IfNotPresent
@@ -705,10 +731,12 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+{% if kube_cilium_nodeinit_enabled == true %}
         - name: cilium-bootstrap-file-dir
           hostPath:
             path: "/tmp/cilium-bootstrap.d"
             type: DirectoryOrCreate
+{% endif %}
             # To read the clustermesh configuration
         - name: clustermesh-secrets
           secret:
@@ -721,6 +749,7 @@ spec:
           configMap:
             name: cilium-config
 
+{% if kube_cilium_nodeinit_enabled == true %}
 ---
 # Source: cilium/templates/cilium-nodeinit/daemonset.yaml
 kind: DaemonSet
@@ -729,18 +758,19 @@ metadata:
   name: cilium-node-init
   namespace: kube-system
   labels:
+    app: cilium-node-init
     app.kubernetes.io/name: cilium-node-init
     app.kubernetes.io/part-of: kube-system
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/name: cilium-node-init
+      app: cilium-node-init
   template:
     metadata:
       annotations:
       labels:
-        app.kubernetes.io/name: cilium-node-init
+        app: cilium-node-init
     spec:
       tolerations:
         - operator: Exists
@@ -789,6 +819,7 @@ spec:
                 mkdir -p "/tmp/cilium-bootstrap.d"
                 date > "/tmp/cilium-bootstrap.d/cilium-bootstrap-time"
                 echo "Node initialization complete"
+{% endif %}
 
 ---
 # Source: cilium/templates/cilium-operator/deployment.yaml
@@ -798,6 +829,8 @@ metadata:
   name: cilium-operator
   namespace: kube-system
   labels:
+    io.cilium/app: operator
+    name: cilium-operator
     app.kubernetes.io/name: cilium-operator
     app.kubernetes.io/part-of: kube-system
     addonmanager.kubernetes.io/mode: Reconcile
@@ -807,7 +840,8 @@ spec:
   replicas: {{ kube_cilium_operator_replicas }}
   selector:
     matchLabels:
-      app.kubernetes.io/name: cilium-operator
+      io.cilium/app: operator
+      name: cilium-operator
   strategy:
     rollingUpdate:
       maxSurge: 1
@@ -817,7 +851,8 @@ spec:
     metadata:
       annotations:
       labels:
-        app.kubernetes.io/name: cilium-operator
+        io.cilium/app: operator
+        name: cilium-operator
     spec:
       # In HA mode, cilium-operator pods must not be scheduled on the same
       # node as they will clash with each other.