|
| 1 | +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"). You may |
| 4 | +// not use this file except in compliance with the License. A copy of the |
| 5 | +// License is located at |
| 6 | +// |
| 7 | +// http://aws.amazon.com/apache2.0/ |
| 8 | +// |
| 9 | +// or in the "license" file accompanying this file. This file is distributed |
| 10 | +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either |
| 11 | +// express or implied. See the License for the specific language governing |
| 12 | +// permissions and limitations under the License. |
| 13 | + |
| 14 | +package crds |
| 15 | + |
| 16 | +import ( |
| 17 | + "context" |
| 18 | + "time" |
| 19 | + |
| 20 | + "github.com/aws/amazon-vpc-resource-controller-k8s/apis/vpcresources/v1alpha1" |
| 21 | + ec2API "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api" |
| 22 | + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/ec2/api/cleanup" |
| 23 | + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config" |
| 24 | + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/k8s" |
| 25 | + "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/utils" |
| 26 | + "github.com/go-logr/logr" |
| 27 | + "github.com/prometheus/client_golang/prometheus" |
| 28 | + v1 "k8s.io/api/core/v1" |
| 29 | + "k8s.io/apimachinery/pkg/api/errors" |
| 30 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 31 | + "k8s.io/apimachinery/pkg/runtime" |
| 32 | + "k8s.io/apimachinery/pkg/types" |
| 33 | + "k8s.io/apimachinery/pkg/util/wait" |
| 34 | + "k8s.io/client-go/util/retry" |
| 35 | + ctrl "sigs.k8s.io/controller-runtime" |
| 36 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 37 | + "sigs.k8s.io/controller-runtime/pkg/controller" |
| 38 | + "sigs.k8s.io/controller-runtime/pkg/metrics" |
| 39 | +) |
| 40 | + |
| 41 | +var ( |
| 42 | + prometheusRegistered = false |
| 43 | + recreateCNINodeCallCount = prometheus.NewCounter( |
| 44 | + prometheus.CounterOpts{ |
| 45 | + Name: "recreate_cniNode_call_count", |
| 46 | + Help: "The number of requests made by controller to recreate CNINode when node exists", |
| 47 | + }, |
| 48 | + ) |
| 49 | + recreateCNINodeErrCount = prometheus.NewCounter( |
| 50 | + prometheus.CounterOpts{ |
| 51 | + Name: "recreate_cniNode_err_count", |
| 52 | + Help: "The number of requests that failed when controller tried to recreate the CNINode", |
| 53 | + }, |
| 54 | + ) |
| 55 | +) |
| 56 | + |
| 57 | +func prometheusRegister() { |
| 58 | + prometheusRegistered = true |
| 59 | + |
| 60 | + metrics.Registry.MustRegister( |
| 61 | + recreateCNINodeCallCount, |
| 62 | + recreateCNINodeErrCount) |
| 63 | + |
| 64 | + prometheusRegistered = true |
| 65 | +} |
| 66 | + |
| 67 | +// CNINodeReconciler reconciles a CNINode object |
| 68 | +type CNINodeReconciler struct { |
| 69 | + client.Client |
| 70 | + Scheme *runtime.Scheme |
| 71 | + Context context.Context |
| 72 | + Log logr.Logger |
| 73 | + EC2Wrapper ec2API.EC2Wrapper |
| 74 | + K8sAPI k8s.K8sWrapper |
| 75 | + ClusterName string |
| 76 | + VPCID string |
| 77 | + FinalizerManager k8s.FinalizerManager |
| 78 | +} |
| 79 | + |
| 80 | +//+kubebuilder:rbac:groups=vpcresources.k8s.aws,resources=cninodes,verbs=get;list;watch;create;update;patch; |
| 81 | + |
| 82 | +// Reconcile handles CNINode create/update/delete events |
| 83 | +// Reconciler will add the finalizer and cluster name tag if it does not exist and finalize on CNINode on deletion to clean up leaked resource on node |
| 84 | +func (r *CNINodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { |
| 85 | + cniNode := &v1alpha1.CNINode{} |
| 86 | + if err := r.Client.Get(ctx, req.NamespacedName, cniNode); err != nil { |
| 87 | + if errors.IsNotFound(err) { |
| 88 | + r.Log.Info("CNINode is deleted", "CNINode", req.NamespacedName) |
| 89 | + } |
| 90 | + // Ignore not found error |
| 91 | + return ctrl.Result{}, client.IgnoreNotFound(err) |
| 92 | + } |
| 93 | + |
| 94 | + nodeFound := true |
| 95 | + node := &v1.Node{} |
| 96 | + if err := r.Client.Get(ctx, req.NamespacedName, node); err != nil { |
| 97 | + if errors.IsNotFound(err) { |
| 98 | + nodeFound = false |
| 99 | + } else { |
| 100 | + r.Log.Error(err, "failed to get the node object in CNINode reconciliation, will retry") |
| 101 | + // Requeue request so it can be retried |
| 102 | + return ctrl.Result{}, err |
| 103 | + } |
| 104 | + } |
| 105 | + |
| 106 | + if cniNode.GetDeletionTimestamp().IsZero() { |
| 107 | + shouldPatch := false |
| 108 | + cniNodeCopy := cniNode.DeepCopy() |
| 109 | + // Add cluster name tag if it does not exist |
| 110 | + val, ok := cniNode.Spec.Tags[config.CNINodeClusterNameKey] |
| 111 | + if !ok || val != r.ClusterName { |
| 112 | + if len(cniNodeCopy.Spec.Tags) != 0 { |
| 113 | + cniNodeCopy.Spec.Tags[config.CNINodeClusterNameKey] = r.ClusterName |
| 114 | + } else { |
| 115 | + cniNodeCopy.Spec.Tags = map[string]string{ |
| 116 | + config.CNINodeClusterNameKey: r.ClusterName, |
| 117 | + } |
| 118 | + } |
| 119 | + shouldPatch = true |
| 120 | + } |
| 121 | + // if node exists, get & add OS label if it does not exist on CNINode |
| 122 | + if nodeFound { |
| 123 | + nodeLabelOS := node.ObjectMeta.Labels[config.NodeLabelOS] |
| 124 | + val, ok = cniNode.ObjectMeta.Labels[config.NodeLabelOS] |
| 125 | + if !ok || val != nodeLabelOS { |
| 126 | + if len(cniNodeCopy.ObjectMeta.Labels) != 0 { |
| 127 | + cniNodeCopy.ObjectMeta.Labels[config.NodeLabelOS] = nodeLabelOS |
| 128 | + } else { |
| 129 | + cniNodeCopy.ObjectMeta.Labels = map[string]string{ |
| 130 | + config.NodeLabelOS: nodeLabelOS, |
| 131 | + } |
| 132 | + } |
| 133 | + shouldPatch = true |
| 134 | + } |
| 135 | + } |
| 136 | + |
| 137 | + if shouldPatch { |
| 138 | + r.Log.Info("patching CNINode to add required fields Tags and Labels", "cninode", cniNode.Name) |
| 139 | + return ctrl.Result{}, r.Client.Patch(ctx, cniNodeCopy, client.MergeFromWithOptions(cniNode, client.MergeFromWithOptimisticLock{})) |
| 140 | + } |
| 141 | + |
| 142 | + // Add finalizer if it does not exist |
| 143 | + if err := r.FinalizerManager.AddFinalizers(ctx, cniNode, config.NodeTerminationFinalizer); err != nil { |
| 144 | + r.Log.Error(err, "failed to add finalizer on CNINode, will retry", "cniNode", cniNode.Name, "finalizer", config.NodeTerminationFinalizer) |
| 145 | + return ctrl.Result{}, err |
| 146 | + } |
| 147 | + return ctrl.Result{}, nil |
| 148 | + |
| 149 | + } else { // CNINode is marked for deletion |
| 150 | + if !nodeFound { |
| 151 | + // node is also deleted, proceed with running the cleanup routine and remove the finalizer |
| 152 | + |
| 153 | + // run cleanup for Linux nodes only |
| 154 | + if val, ok := cniNode.ObjectMeta.Labels[config.NodeLabelOS]; ok && val == config.OSLinux { |
| 155 | + r.Log.Info("running the finalizer routine on cniNode", "cniNode", cniNode.Name) |
| 156 | + cleaner := &cleanup.NodeTerminationCleaner{ |
| 157 | + NodeName: cniNode.Name, |
| 158 | + } |
| 159 | + cleaner.ENICleaner = &cleanup.ENICleaner{ |
| 160 | + EC2Wrapper: r.EC2Wrapper, |
| 161 | + Manager: cleaner, |
| 162 | + VPCID: r.VPCID, |
| 163 | + Log: ctrl.Log.WithName("eniCleaner").WithName("node"), |
| 164 | + } |
| 165 | + |
| 166 | + if err := cleaner.DeleteLeakedResources(); err != nil { |
| 167 | + r.Log.Error(err, "failed to cleanup resources during node termination, request will be requeued") |
| 168 | + // Return err if failed to delete leaked ENIs on node so it can be retried |
| 169 | + return ctrl.Result{}, err |
| 170 | + } |
| 171 | + } |
| 172 | + |
| 173 | + if err := r.FinalizerManager.RemoveFinalizers(ctx, cniNode, config.NodeTerminationFinalizer); err != nil { |
| 174 | + r.Log.Error(err, "failed to remove finalizer on CNINode, will retry", "cniNode", cniNode.Name, "finalizer", config.NodeTerminationFinalizer) |
| 175 | + return ctrl.Result{}, err |
| 176 | + } |
| 177 | + return ctrl.Result{}, nil |
| 178 | + } else { |
| 179 | + // node exists, do not run the cleanup routine(periodic cleanup routine will anyway delete leaked ENIs), remove the finalizer |
| 180 | + // to proceed with object deletion, and recreate similar object |
| 181 | + |
| 182 | + // Create a copy without deletion timestamp for creation |
| 183 | + newCNINode := &v1alpha1.CNINode{ |
| 184 | + ObjectMeta: metav1.ObjectMeta{ |
| 185 | + Name: cniNode.Name, |
| 186 | + Namespace: "", |
| 187 | + OwnerReferences: cniNode.OwnerReferences, |
| 188 | + // TODO: should we include finalizers at object creation or let controller patch it on Create/Update event? |
| 189 | + Finalizers: cniNode.Finalizers, |
| 190 | + }, |
| 191 | + Spec: cniNode.Spec, |
| 192 | + } |
| 193 | + |
| 194 | + if err := r.FinalizerManager.RemoveFinalizers(ctx, cniNode, config.NodeTerminationFinalizer); err != nil { |
| 195 | + r.Log.Error(err, "failed to remove finalizer on CNINode on node deletion, will retry") |
| 196 | + return ctrl.Result{}, err |
| 197 | + } |
| 198 | + // wait till CNINode is deleted before recreation as the new object will be created with same name to avoid "object already exists" error |
| 199 | + if err := r.waitTillCNINodeDeleted(k8s.NamespacedName(newCNINode)); err != nil { |
| 200 | + // raise event if CNINode could not be deleted after removing the finalizer |
| 201 | + r.K8sAPI.BroadcastEvent(cniNode, utils.CNINodeDeleteFailed, "CNINode deletion failed and object could not be recreated by the vpc-resource-controller, will retry", |
| 202 | + v1.EventTypeWarning) |
| 203 | + // requeue here to check if CNINode deletion is successful and retry CNINode deletion if node exists |
| 204 | + return ctrl.Result{}, err |
| 205 | + } |
| 206 | + |
| 207 | + r.Log.Info("creating CNINode after it has been deleted as node still exists", "cniNode", newCNINode.Name) |
| 208 | + recreateCNINodeCallCount.Inc() |
| 209 | + if err := r.createCNINodeFromObj(ctx, newCNINode); err != nil { |
| 210 | + recreateCNINodeErrCount.Inc() |
| 211 | + // raise event on node publish warning that CNINode is deleted and could not be recreated by controller |
| 212 | + utils.SendNodeEventWithNodeName(r.K8sAPI, node.Name, utils.CNINodeCreateFailed, |
| 213 | + "CNINode was deleted and failed to be recreated by the vpc-resource-controller", v1.EventTypeWarning, r.Log) |
| 214 | + // return nil as deleted and we cannot recreate the object now |
| 215 | + return ctrl.Result{}, nil |
| 216 | + } |
| 217 | + r.Log.Info("successfully recreated CNINode", "cniNode", newCNINode.Name) |
| 218 | + } |
| 219 | + } |
| 220 | + return ctrl.Result{}, nil |
| 221 | +} |
| 222 | + |
| 223 | +// SetupWithManager sets up the controller with the Manager. |
| 224 | +func (r *CNINodeReconciler) SetupWithManager(mgr ctrl.Manager) error { |
| 225 | + if !prometheusRegistered { |
| 226 | + prometheusRegister() |
| 227 | + } |
| 228 | + return ctrl.NewControllerManagedBy(mgr). |
| 229 | + For(&v1alpha1.CNINode{}). |
| 230 | + WithOptions(controller.Options{MaxConcurrentReconciles: config.MaxNodeConcurrentReconciles}). |
| 231 | + Complete(r) |
| 232 | +} |
| 233 | + |
| 234 | +// waitTillCNINodeDeleted waits for CNINode to be deleted with timeout and returns error |
| 235 | +func (r *CNINodeReconciler) waitTillCNINodeDeleted(nameSpacedCNINode types.NamespacedName) error { |
| 236 | + oldCNINode := &v1alpha1.CNINode{} |
| 237 | + |
| 238 | + return wait.PollUntilContextTimeout(context.TODO(), 500*time.Millisecond, time.Second*3, true, func(ctx context.Context) (bool, error) { |
| 239 | + if err := r.Client.Get(ctx, nameSpacedCNINode, oldCNINode); err != nil && errors.IsNotFound(err) { |
| 240 | + return true, nil |
| 241 | + } |
| 242 | + return false, nil |
| 243 | + }) |
| 244 | +} |
| 245 | + |
| 246 | +// createCNINodeFromObj will create CNINode with backoff and returns error if CNINode is not recreated |
| 247 | +func (r *CNINodeReconciler) createCNINodeFromObj(ctx context.Context, newCNINode client.Object) error { |
| 248 | + return retry.OnError(retry.DefaultBackoff, func(error) bool { return true }, |
| 249 | + func() error { |
| 250 | + return r.Client.Create(ctx, newCNINode) |
| 251 | + }) |
| 252 | +} |
![dependabot[bot]](https://avatars.githubusercontent.com/in/29110?v=4&size=40){kind=avatar}
0 commit comments