Skip to content

Commit 058a0e2

Browse files
yash97yash97
authored
Merge branch 'master' into sync
2 parents 6c172e3 + a4bf191 commit 058a0e2

28 files changed

+1830
-230
lines changed

‎controllers/core/configmap_controller.go

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ type ConfigMapReconciler struct {
4646
Condition condition.Conditions
4747
curWinIPAMEnabledCond bool
4848
curWinPrefixDelegationEnabledCond bool
49-
curWinPDWarmIPTarget int
50-
curWinPDMinIPTarget int
49+
curWinWarmIPTarget int
50+
curWinMinIPTarget int
5151
curWinPDWarmPrefixTarget int
5252
Context context.Context
5353
}
@@ -116,21 +116,33 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
116116
isPrefixFlagUpdated = true
117117
}
118118

119-
// Check if configurations for Windows prefix delegation have changed
120-
var isPDConfigUpdated bool
121-
warmIPTarget, minIPTarget, warmPrefixTarget := config.ParseWinPDTargets(r.Log, configmap)
122-
if r.curWinPDWarmIPTarget != warmIPTarget || r.curWinPDMinIPTarget != minIPTarget || r.curWinPDWarmPrefixTarget != warmPrefixTarget {
123-
r.curWinPDWarmIPTarget = warmIPTarget
124-
r.curWinPDMinIPTarget = minIPTarget
119+
// Check if Windows IP target configurations in ConfigMap have changed
120+
var isWinIPConfigsUpdated bool
121+
122+
warmIPTarget, minIPTarget, warmPrefixTarget, isPDEnabled := config.ParseWinIPTargetConfigs(r.Log, configmap)
123+
var winMinIPTargetUpdated = r.curWinMinIPTarget != minIPTarget
124+
var winWarmIPTargetUpdated = r.curWinWarmIPTarget != warmIPTarget
125+
var winPDWarmPrefixTargetUpdated = r.curWinPDWarmPrefixTarget != warmPrefixTarget
126+
if winWarmIPTargetUpdated || winMinIPTargetUpdated {
127+
r.curWinWarmIPTarget = warmIPTarget
128+
r.curWinMinIPTarget = minIPTarget
129+
isWinIPConfigsUpdated = true
130+
}
131+
if isPDEnabled && winPDWarmPrefixTargetUpdated {
125132
r.curWinPDWarmPrefixTarget = warmPrefixTarget
126-
logger.Info("updated PD configs from configmap", config.WarmIPTarget, r.curWinPDWarmIPTarget,
127-
config.MinimumIPTarget, r.curWinPDMinIPTarget, config.WarmPrefixTarget, r.curWinPDWarmPrefixTarget)
128-
129-
isPDConfigUpdated = true
133+
isWinIPConfigsUpdated = true
134+
}
135+
if isWinIPConfigsUpdated {
136+
logger.Info(
137+
"Detected update in Windows IP configuration parameter values in ConfigMap",
138+
config.WinWarmIPTarget, r.curWinWarmIPTarget,
139+
config.WinMinimumIPTarget, r.curWinMinIPTarget,
140+
config.WinWarmPrefixTarget, r.curWinPDWarmPrefixTarget,
141+
config.EnableWindowsPrefixDelegationKey, isPDEnabled,
142+
)
130143
}
131144

132-
// Flag is updated, update all nodes
133-
if isIPAMFlagUpdated || isPrefixFlagUpdated || isPDConfigUpdated {
145+
if isIPAMFlagUpdated || isPrefixFlagUpdated || isWinIPConfigsUpdated {
134146
err := UpdateNodesOnConfigMapChanges(r.K8sAPI, r.NodeManager)
135147
if err != nil {
136148
// Error in updating nodes

‎controllers/core/configmap_controller_test.go

Lines changed: 49 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,9 @@ package controllers
1616
import (
1717
"context"
1818
"errors"
19+
"strconv"
1920
"testing"
2021

21-
mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
22-
mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
23-
mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
24-
mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
25-
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
26-
cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
2722
"github.com/golang/mock/gomock"
2823
"github.com/stretchr/testify/assert"
2924
corev1 "k8s.io/api/core/v1"
@@ -35,18 +30,36 @@ import (
3530
fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
3631
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3732
"sigs.k8s.io/controller-runtime/pkg/reconcile"
33+
34+
mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
35+
mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
36+
mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
37+
mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
38+
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
39+
cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
3840
)
3941

4042
var (
4143
mockConfigMap = &corev1.ConfigMap{
4244
TypeMeta: metav1.TypeMeta{},
4345
ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
44-
Data: map[string]string{config.EnableWindowsIPAMKey: "true", config.EnableWindowsPrefixDelegationKey: "true"},
46+
Data: map[string]string{
47+
config.EnableWindowsIPAMKey: "true",
48+
config.EnableWindowsPrefixDelegationKey: "false",
49+
config.WinMinimumIPTarget: strconv.Itoa(config.IPv4DefaultWinMinIPTarget),
50+
config.WinWarmIPTarget: strconv.Itoa(config.IPv4DefaultWinWarmIPTarget),
51+
},
4552
}
4653
mockConfigMapPD = &corev1.ConfigMap{
4754
TypeMeta: metav1.TypeMeta{},
4855
ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
49-
Data: map[string]string{config.EnableWindowsIPAMKey: "false", config.EnableWindowsPrefixDelegationKey: "true"},
56+
Data: map[string]string{
57+
config.EnableWindowsIPAMKey: "true",
58+
config.EnableWindowsPrefixDelegationKey: "true",
59+
config.WinMinimumIPTarget: strconv.Itoa(config.IPv4PDDefaultMinIPTargetSize),
60+
config.WinWarmIPTarget: strconv.Itoa(config.IPv4PDDefaultWarmIPTargetSize),
61+
config.WinWarmPrefixTarget: strconv.Itoa(config.IPv4PDDefaultWarmPrefixTargetSize),
62+
},
5063
}
5164
mockConfigMapReq = reconcile.Request{
5265
NamespacedName: types.NamespacedName{
@@ -89,11 +102,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
89102
return ConfigMapMock{
90103
MockNodeManager: mockNodeManager,
91104
ConfigMapReconciler: &ConfigMapReconciler{
92-
Client: client,
93-
Log: zap.New(),
94-
NodeManager: mockNodeManager,
95-
K8sAPI: mockK8sWrapper,
96-
Condition: mockCondition,
105+
Client: client,
106+
Log: zap.New(),
107+
NodeManager: mockNodeManager,
108+
K8sAPI: mockK8sWrapper,
109+
Condition: mockCondition,
110+
curWinMinIPTarget: config.IPv4DefaultWinMinIPTarget,
111+
curWinWarmIPTarget: config.IPv4DefaultWinWarmIPTarget,
97112
},
98113
MockNode: mockNode,
99114
MockK8sAPI: mockK8sWrapper,
@@ -103,13 +118,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
103118
}
104119
}
105120

106-
func Test_Reconcile_ConfigMap_Updated(t *testing.T) {
121+
func Test_Reconcile_ConfigMap_Updated_Secondary_IP(t *testing.T) {
107122
ctrl := gomock.NewController(t)
108123
defer ctrl.Finish()
109124

110125
mock := NewConfigMapMock(ctrl, mockConfigMap)
111126
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(true)
112-
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true)
127+
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
113128
mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil)
114129
mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true)
115130
mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil)
@@ -120,14 +135,32 @@ func Test_Reconcile_ConfigMap_Updated(t *testing.T) {
120135
res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq)
121136
assert.NoError(t, err)
122137
assert.Equal(t, res, reconcile.Result{})
138+
}
123139

140+
func Test_Reconcile_ConfigMap_Updated_PD(t *testing.T) {
141+
ctrl := gomock.NewController(t)
142+
defer ctrl.Finish()
143+
144+
mock := NewConfigMapMock(ctrl, mockConfigMapPD)
145+
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(true)
146+
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true)
147+
mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil)
148+
mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true)
149+
mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil)
150+
151+
mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()
152+
153+
cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown"))
154+
res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq)
155+
assert.NoError(t, err)
156+
assert.Equal(t, res, reconcile.Result{})
124157
}
125158

126159
func Test_Reconcile_ConfigMap_PD_Disabled_If_IPAM_Disabled(t *testing.T) {
127160
ctrl := gomock.NewController(t)
128161
defer ctrl.Finish()
129162

130-
mock := NewConfigMapMock(ctrl, mockConfigMapPD)
163+
mock := NewConfigMapMock(ctrl, mockConfigMap)
131164
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false)
132165
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
133166
mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()

‎controllers/custom/builder.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,15 +113,16 @@ func (b *Builder) Complete(reconciler Reconciler) (healthz.Checker, error) {
113113
workqueue.DefaultControllerRateLimiter(), b.options.Name)
114114

115115
optimizedListWatch := newOptimizedListWatcher(b.ctx, b.clientSet.CoreV1().RESTClient(),
116-
b.converter.Resource(), b.options.Namespace, b.options.PageLimit, b.converter)
116+
b.converter.Resource(), b.options.Namespace, b.converter, b.log.WithName("listWatcher"))
117117

118118
// Create the config for low level controller with the custom converter
119119
// list and watch
120120
config := &cache.Config{
121-
Queue: cache.NewDeltaFIFO(b.converter.Indexer, b.dataStore),
122-
ListerWatcher: optimizedListWatch,
123-
ObjectType: b.converter.ResourceType(),
124-
FullResyncPeriod: b.options.ResyncPeriod,
121+
Queue: cache.NewDeltaFIFO(b.converter.Indexer, b.dataStore),
122+
ListerWatcher: optimizedListWatch,
123+
WatchListPageSize: int64(b.options.PageLimit),
124+
ObjectType: b.converter.ResourceType(),
125+
FullResyncPeriod: b.options.ResyncPeriod,
125126
Process: func(obj interface{}, _ bool) error {
126127
// from oldest to newest
127128
for _, d := range obj.(cache.Deltas) {

‎controllers/custom/custom_controller.go

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/condition"
2323
"github.com/go-logr/logr"
24+
apierrors "k8s.io/apimachinery/pkg/api/errors"
2425
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2526
"k8s.io/apimachinery/pkg/runtime"
2627
"k8s.io/apimachinery/pkg/types"
@@ -178,23 +179,26 @@ func (c *CustomController) WaitForCacheSync(controller cache.Controller) {
178179

179180
// newOptimizedListWatcher returns a list watcher with a custom list function that converts the
180181
// response for each page using the converter function and returns a general watcher
181-
func newOptimizedListWatcher(ctx context.Context, restClient cache.Getter, resource string, namespace string, limit int,
182-
converter Converter) *cache.ListWatch {
182+
func newOptimizedListWatcher(ctx context.Context, restClient cache.Getter, resource string, namespace string,
183+
converter Converter, log logr.Logger) *cache.ListWatch {
183184

184185
listFunc := func(options metav1.ListOptions) (runtime.Object, error) {
185186
list, err := restClient.Get().
186187
Namespace(namespace).
187188
Resource(resource).
188-
// This needs to be done because just setting the limit using option's
189-
// Limit is being overridden and the response is returned without pagination.
190189
VersionedParams(&metav1.ListOptions{
191-
Limit: int64(limit),
190+
Limit: options.Limit,
192191
Continue: options.Continue,
193192
}, metav1.ParameterCodec).
194193
Do(ctx).
195194
Get()
196195
if err != nil {
197-
return list, err
196+
if statusErr, ok := err.(*apierrors.StatusError); ok {
197+
log.Error(err, "List operation error", "code", statusErr.Status().Code)
198+
} else {
199+
log.Error(err, "List operation error")
200+
}
201+
return nil, err
198202
}
199203
// Strip down the the list before passing the paginated response back to
200204
// the pager function
@@ -206,11 +210,20 @@ func newOptimizedListWatcher(ctx context.Context, restClient cache.Getter, resou
206210
// before storing the object in the data store.
207211
watchFunc := func(options metav1.ListOptions) (watch.Interface, error) {
208212
options.Watch = true
209-
return restClient.Get().
213+
watch, err := restClient.Get().
210214
Namespace(namespace).
211215
Resource(resource).
212216
VersionedParams(&options, metav1.ParameterCodec).
213217
Watch(ctx)
218+
if err != nil {
219+
if statusErr, ok := err.(*apierrors.StatusError); ok {
220+
log.Error(err, "Watch operation error", "code", statusErr.Status().Code)
221+
} else {
222+
log.Error(err, "Watch operation error")
223+
}
224+
return nil, err
225+
}
226+
return watch, err
214227
}
215228
return &cache.ListWatch{ListFunc: listFunc, WatchFunc: watchFunc}
216229
}

‎docs/windows/secondary_ip_mode_config_options.md

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Configuration options when using secondary IP addresses Windows
2+
3+
We provide multiple configuration options that allow you to fine-tune the IP address allocation behavior on Windows
4+
nodes using the secondary IP address mode. These configuration options can be set in the `amazon-vpc-cni` ConfigMap in
5+
the `kube-system` namespace.
6+
7+
- `windows-warm-ip-target` → The total number of IP addresses that should be allocated to each Windows node in excess of
8+
the current need at any given time. The excess IPs can be used by newly launched pods, which aids in faster pod
9+
startup times since there is no wait time for additional IP addresses to be allocated. The VPC Resource Controller
10+
will attempt to ensure that this excess desired threshold is always met.
11+
12+
Defaults to 3 if unspecified or invalid. Must be greater than or equal to 1.
13+
14+
For example, if no pods were running on a given Windows node, and if you set `windows-warm-ip-target` to 5, the VPC
15+
Resource Controller will aim to ensure that each Windows node always has at least 5 IP addresses in excess, ready for
16+
use, allocated to its ENI. If 2 pods are scheduled on the node, the controller will allocate 2 additional IP addresses
17+
to the ENI, maintaining the 5 warm IP address target.
18+
19+
- `windows-minimum-ip-target` → Defaults to 3 if unspecified or invalid. The minimum number of IP addresses, both in use
20+
by running pods and available as warm IPs, that should be allocated to each Windows node at any given time. The
21+
controller will attempt to ensure that this minimum threshold is always met.
22+
23+
Defaults to 3 if unspecified or invalid. Must be greater than or equal to 0.
24+
25+
For example, if no pods were running on a given Windows node, and if you set `windows-minimum-ip-target` to 10, the
26+
VPC Resource Controller will aim to ensure that the total number of IP addresses on the Windows node should be at
27+
least 10. Therefore, before pods are scheduled, there should be at least 10 IP addresses available. If 5 pods are
28+
scheduled on a given node, they will consume 5 of the 10 available IPs. The VPC Resource Controller will keep 5 the
29+
remaining available IPs available in addition to the 5 already in use to meet the target of 10.
30+
31+
### Considerations while using the above configuration options
32+
33+
- These configuration options only apply when the VPC Resource Controller is operating in the secondary IP mode. They do
34+
not affect the prefix delegation mode. More explicitly, if `enable-windows-prefix-delegation` is set to false, or is
35+
not specified, then the VPC Resource Controller operates in secondary IP mode.
36+
- Setting either `windows-warm-ip-target` or `windows-minimum-ip-target` to a negative value will result in the
37+
respective default value being used.
38+
- If the values of `windows-warm-ip-target` or `windows-minimum-ip-target` are set such that the maximum node IP
39+
capacity would be exceeded, the controller will limit the allocation to the maximum capacity possible.
40+
- The `warm-prefix-target` configuration option will be ignored when using the secondary IP mode, as it only applies to
41+
the prefix delegation mode.
42+
- If `windows-warm-ip-target` is set to 0, the system will implicitly set `windows-warm-ip-target` to 1. This is
43+
because on-demand IP allocation whereby an IP is allocated on the Windows node as the pods are scheduled is currently
44+
not supported. Implicitly Setting `windows-warm-ip-target` to 1 ensures the minimum acceptable non-zero value is set
45+
since the `windows-warm-ip-target` should always be at least 1.
46+
- The configuration options `warm-ip-target` and `minimum-ip-target` are deprecated in favor of the new
47+
options `windows-warm-ip-target` and `windows-minimum-ip-target`.
48+
49+
### Examples
50+
51+
| `windows-warm-ip-target` | `windows-minimum-ip-target` | Running Pods | Total Allocated IPs | Warm IPs |
52+
|--------------------------|-----------------------------|--------------|---------------------|----------|
53+
| 1 | 0 | 0 | 1 | 1 |
54+
| 1 | 0 | 5 | 6 | 1 |
55+
| 5 | 0 | 0 | 5 | 5 |
56+
| 1 | 1 | 0 | 1 | 1 |
57+
| 1 | 1 | 1 | 2 | 1 |
58+
| 1 | 3 | 3 | 4 | 1 |
59+
| 1 | 3 | 5 | 6 | 1 |
60+
| 5 | 10 | 0 | 10 | 10 |
61+
| 10 | 10 | 0 | 10 | 10 |
62+
| 10 | 10 | 10 | 20 | 10 |
63+
| 15 | 10 | 10 | 25 | 15 |

0 commit comments

Comments
 (0)