Skip to content

Commit 1e9cb9b

Browse files
committed
Add Windows secondary IP mode configurable options (#443)
#443
1 parent 964a3cc commit 1e9cb9b

File tree

17 files changed

+1225
-156
lines changed

17 files changed

+1225
-156
lines changed

controllers/core/configmap_controller.go

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ type ConfigMapReconciler struct {
4646
Condition condition.Conditions
4747
curWinIPAMEnabledCond bool
4848
curWinPrefixDelegationEnabledCond bool
49-
curWinPDWarmIPTarget int
50-
curWinPDMinIPTarget int
49+
curWinWarmIPTarget int
50+
curWinMinIPTarget int
5151
curWinPDWarmPrefixTarget int
5252
Context context.Context
5353
}
@@ -116,21 +116,34 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
116116
isPrefixFlagUpdated = true
117117
}
118118

119-
// Check if configurations for Windows prefix delegation have changed
120-
var isPDConfigUpdated bool
121-
warmIPTarget, minIPTarget, warmPrefixTarget := config.ParseWinPDTargets(r.Log, configmap)
122-
if r.curWinPDWarmIPTarget != warmIPTarget || r.curWinPDMinIPTarget != minIPTarget || r.curWinPDWarmPrefixTarget != warmPrefixTarget {
123-
r.curWinPDWarmIPTarget = warmIPTarget
124-
r.curWinPDMinIPTarget = minIPTarget
119+
// Check if Windows IP target configurations in ConfigMap have changed
120+
var isWinIPConfigsUpdated bool
121+
122+
warmIPTarget, minIPTarget, warmPrefixTarget, isPDEnabled := config.ParseWinIPTargetConfigs(r.Log, configmap)
123+
var winMinIPTargetUpdated = r.curWinMinIPTarget != minIPTarget
124+
var winWarmIPTargetUpdated = r.curWinWarmIPTarget != warmIPTarget
125+
var winPDWarmPrefixTargetUpdated = r.curWinPDWarmPrefixTarget != warmPrefixTarget
126+
if winWarmIPTargetUpdated || winMinIPTargetUpdated {
127+
r.curWinWarmIPTarget = warmIPTarget
128+
r.curWinMinIPTarget = minIPTarget
129+
isWinIPConfigsUpdated = true
130+
}
131+
if isPDEnabled && winPDWarmPrefixTargetUpdated {
125132
r.curWinPDWarmPrefixTarget = warmPrefixTarget
126-
logger.Info("updated PD configs from configmap", config.WarmIPTarget, r.curWinPDWarmIPTarget,
127-
config.MinimumIPTarget, r.curWinPDMinIPTarget, config.WarmPrefixTarget, r.curWinPDWarmPrefixTarget)
128-
129-
isPDConfigUpdated = true
133+
isWinIPConfigsUpdated = true
134+
}
135+
if isWinIPConfigsUpdated {
136+
logger.Info(
137+
"Detected update in Windows IP configuration parameter values in ConfigMap",
138+
config.WinWarmIPTarget, r.curWinWarmIPTarget,
139+
config.WinMinimumIPTarget, r.curWinMinIPTarget,
140+
config.WinWarmPrefixTarget, r.curWinPDWarmPrefixTarget,
141+
config.EnableWindowsPrefixDelegationKey, isPDEnabled,
142+
)
130143
}
131144

132-
// Flag is updated, update all nodes
133-
if isIPAMFlagUpdated || isPrefixFlagUpdated || isPDConfigUpdated {
145+
var nodesRequireUpdate = isIPAMFlagUpdated || isPrefixFlagUpdated || isWinIPConfigsUpdated
146+
if nodesRequireUpdate {
134147
err := UpdateNodesOnConfigMapChanges(r.K8sAPI, r.NodeManager)
135148
if err != nil {
136149
// Error in updating nodes

controllers/core/configmap_controller_test.go

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,9 @@ package controllers
1616
import (
1717
"context"
1818
"errors"
19+
"strconv"
1920
"testing"
2021

21-
mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
22-
mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
23-
mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
24-
mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
25-
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
26-
cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
2722
"github.com/golang/mock/gomock"
2823
"github.com/stretchr/testify/assert"
2924
corev1 "k8s.io/api/core/v1"
@@ -35,18 +30,35 @@ import (
3530
fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
3631
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3732
"sigs.k8s.io/controller-runtime/pkg/reconcile"
33+
34+
mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
35+
mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
36+
mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
37+
mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
38+
"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
39+
cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
3840
)
3941

4042
var (
4143
mockConfigMap = &corev1.ConfigMap{
4244
TypeMeta: metav1.TypeMeta{},
4345
ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
44-
Data: map[string]string{config.EnableWindowsIPAMKey: "true", config.EnableWindowsPrefixDelegationKey: "true"},
46+
Data: map[string]string{
47+
config.EnableWindowsIPAMKey: "true",
48+
config.EnableWindowsPrefixDelegationKey: "true",
49+
config.WinMinimumIPTarget: strconv.Itoa(config.IPv4DefaultWinMinIPTarget),
50+
config.WinWarmIPTarget: strconv.Itoa(config.IPv4DefaultWinWarmIPTarget),
51+
},
4552
}
4653
mockConfigMapPD = &corev1.ConfigMap{
4754
TypeMeta: metav1.TypeMeta{},
4855
ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
49-
Data: map[string]string{config.EnableWindowsIPAMKey: "false", config.EnableWindowsPrefixDelegationKey: "true"},
56+
Data: map[string]string{
57+
config.EnableWindowsIPAMKey: "false",
58+
config.EnableWindowsPrefixDelegationKey: "true",
59+
config.WinMinimumIPTarget: strconv.Itoa(config.IPv4PDDefaultMinIPTargetSize),
60+
config.WinWarmIPTarget: strconv.Itoa(config.IPv4PDDefaultWarmIPTargetSize),
61+
},
5062
}
5163
mockConfigMapReq = reconcile.Request{
5264
NamespacedName: types.NamespacedName{
@@ -89,11 +101,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
89101
return ConfigMapMock{
90102
MockNodeManager: mockNodeManager,
91103
ConfigMapReconciler: &ConfigMapReconciler{
92-
Client: client,
93-
Log: zap.New(),
94-
NodeManager: mockNodeManager,
95-
K8sAPI: mockK8sWrapper,
96-
Condition: mockCondition,
104+
Client: client,
105+
Log: zap.New(),
106+
NodeManager: mockNodeManager,
107+
K8sAPI: mockK8sWrapper,
108+
Condition: mockCondition,
109+
curWinMinIPTarget: config.IPv4DefaultWinMinIPTarget,
110+
curWinWarmIPTarget: config.IPv4DefaultWinWarmIPTarget,
97111
},
98112
MockNode: mockNode,
99113
MockK8sAPI: mockK8sWrapper,
@@ -127,7 +141,7 @@ func Test_Reconcile_ConfigMap_PD_Disabled_If_IPAM_Disabled(t *testing.T) {
127141
ctrl := gomock.NewController(t)
128142
defer ctrl.Finish()
129143

130-
mock := NewConfigMapMock(ctrl, mockConfigMapPD)
144+
mock := NewConfigMapMock(ctrl, mockConfigMap)
131145
mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false)
132146
mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
133147
mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Configuration options when using secondary IP addresses Windows
2+
3+
We provide multiple configuration options that allow you to fine-tune the IP address allocation behavior on Windows
4+
nodes using the secondary IP address mode. These configuration options can be set in the `amazon-vpc-cni` ConfigMap in
5+
the `kube-system` namespace.
6+
7+
- `windows-warm-ip-target` → Defaults to 3. The total number of IP addresses that should be allocated
8+
to each Windows node in excess of the current need at any given time. The excess IPs can be used by newly launched
9+
pods, which aids in faster pod startup times since there is no wait time for additional IP addresses to be allocated.
10+
The VPC Resource Controller will attempt to ensure that this excess desired threshold is always met.
11+
12+
For example, if no pods were running on a given Windows node, and if you set `windows-warm-ip-target` to 5, the VPC
13+
Resource Controller will aim to ensure that each Windows node always has at least 5 IP addresses in excess, ready for
14+
use, allocated to its ENI. If 2 pods are scheduled on the node, the controller will allocate 2 additional IP addresses
15+
to the ENI, maintaining the 5 warm IP address target.
16+
17+
- `windows-minimum-ip-target` → Defaults to 3. The minimum number of IP addresses, both in use by
18+
running pods and available as warm IPs, that should be allocated to each Windows node at any given time. The
19+
controller will attempt to ensure that this minimum threshold is always met.
20+
21+
For example, if no pods were running on a given Windows node, and if you set `windows-minimum-ip-target` to 10, the
22+
VPC Resource Controller will aim to ensure that the total number of IP addresses on the Windows node should be at
23+
least 10. Therefore, before pods are scheduled, there should be at least 10 IP addresses available. If 5 pods are
24+
scheduled on a given node, they will consume 5 of the 10 available IPs. The VPC Resource Controller will keep 5 the
25+
remaining available IPs available in addition to the 5 already in use to meet the target ofg 10.
26+
27+
### Considerations while using the above configuration options
28+
29+
- These configuration options only apply when the VPC Resource Controller is operating in the secondary IP mode. They do
30+
not affect the prefix delegation mode. More explicitly, if `enable-windows-prefix-delegation` is set to false, or is
31+
not specified, then the VPC Resource Controller operates in secondary IP mode.
32+
- Setting either `windows-warm-ip-target` or `windows-minimum-ip-target` to a negative value will result in the default
33+
values being used.
34+
- If the values of `windows-warm-ip-target` or `windows-minimum-ip-target` are set such that the maximum node IP
35+
capacity would be exceeded, the controller will limit the allocation to the maximum capacity.
36+
- The warm-prefix-target configuration option will be ignored when using the secondary IP mode, as it is specific to the
37+
prefix delegation mode.
38+
- When both `windows-warm-ip-target` and `windows-minimum-ip-target` are set to 0, the default values will be used. If
39+
only one of the configuration options is set, the other will default to the respective default value. On-demand
40+
allocation of having both `windows-warm-ip-target` and `windows-minimum-ip-target` set to zero with no warm IP
41+
addresses is currently not supported.
42+
- The minimum acceptable value for `windows-warm-ip-target` is 1. The minimum acceptable value
43+
for `windows-minimum-ip-target` is 0, but it will implicitly be set to 1 since the windows-warm-ip-target should
44+
always be at least 1.
45+
- The configuration options `warm-ip-target` and `minimum-ip-target` are deprecated in favor of the new
46+
options `windows-warm-ip-target` and `windows-minimum-ip-target`.
47+
48+
### Examples
49+
50+
| `windows-warm-ip-target` | `windows-minimum-ip-target` | Running Pods | Total Allocated IPs | Warm IPs |
51+
|--------------------------|-----------------------------|--------------|---------------------|----------|
52+
| 1 | 0 | 0 | 1 | 1 |
53+
| 1 | 0 | 5 | 6 | 1 |
54+
| 5 | 0 | 0 | 5 | 5 |
55+
| 1 | 1 | 0 | 1 | 1 |
56+
| 1 | 1 | 1 | 2 | 1 |
57+
| 1 | 3 | 3 | 4 | 1 |
58+
| 1 | 3 | 5 | 6 | 1 |
59+
| 5 | 10 | 0 | 10 | 10 |
60+
| 10 | 10 | 0 | 10 | 10 |
61+
| 10 | 10 | 10 | 20 | 10 |
62+
| 15 | 10 | 10 | 25 | 15 |

pkg/config/loader.go

Lines changed: 83 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@ const (
2828
// Default Configuration for Pod ENI resource type
2929
PodENIDefaultWorker = 30
3030

31-
// Default Configuration for IPv4 resource type
32-
IPv4DefaultWorker = 2
33-
IPv4DefaultWPSize = 3
34-
IPv4DefaultMaxDev = 1
35-
IPv4DefaultResSize = 0
36-
37-
// Default Configuration for IPv4 prefix resource type
31+
// Default Windows Configuration for IPv4 resource type
32+
IPv4DefaultWinWorkerCount = 2
33+
IPv4DefaultWinWarmIPTarget = 3
34+
IPv4DefaultWinMinIPTarget = 3
35+
IPv4DefaultWinMaxDev = 0
36+
IPv4DefaultWinResSize = 0
37+
38+
// Default Windows Configuration for IPv4 prefix resource type
3839
IPv4PDDefaultWorker = 2
3940
IPv4PDDefaultWPSize = 1
4041
IPv4PDDefaultMaxDev = 0
@@ -70,26 +71,45 @@ func LoadResourceConfig() map[string]ResourceConfig {
7071
func LoadResourceConfigFromConfigMap(log logr.Logger, vpcCniConfigMap *v1.ConfigMap) map[string]ResourceConfig {
7172
resourceConfig := getDefaultResourceConfig()
7273

73-
warmIPTarget, minIPTarget, warmPrefixTarget := ParseWinPDTargets(log, vpcCniConfigMap)
74+
warmIPTarget, minIPTarget, warmPrefixTarget, isPDEnabled := ParseWinIPTargetConfigs(log, vpcCniConfigMap)
7475

7576
// If no PD configuration is set in configMap or none is valid, return default resource config
7677
if warmIPTarget == 0 && minIPTarget == 0 && warmPrefixTarget == 0 {
7778
return resourceConfig
7879
}
7980

80-
resourceConfig[ResourceNameIPAddressFromPrefix].WarmPoolConfig.WarmIPTarget = warmIPTarget
81-
resourceConfig[ResourceNameIPAddressFromPrefix].WarmPoolConfig.MinIPTarget = minIPTarget
82-
resourceConfig[ResourceNameIPAddressFromPrefix].WarmPoolConfig.WarmPrefixTarget = warmPrefixTarget
81+
if isPDEnabled {
82+
resourceConfig[ResourceNameIPAddressFromPrefix].WarmPoolConfig.WarmIPTarget = warmIPTarget
83+
resourceConfig[ResourceNameIPAddressFromPrefix].WarmPoolConfig.MinIPTarget = minIPTarget
84+
resourceConfig[ResourceNameIPAddressFromPrefix].WarmPoolConfig.WarmPrefixTarget = warmPrefixTarget
85+
} else {
86+
resourceConfig[ResourceNameIPAddress].WarmPoolConfig.WarmIPTarget = warmIPTarget
87+
resourceConfig[ResourceNameIPAddress].WarmPoolConfig.MinIPTarget = minIPTarget
88+
resourceConfig[ResourceNameIPAddress].WarmPoolConfig.WarmPrefixTarget = warmPrefixTarget // ignore warm prefix in secondary IP mode
89+
}
8390

8491
return resourceConfig
8592
}
8693

87-
// ParseWinPDTargets parses config map for Windows prefix delegation configurations set by users
88-
func ParseWinPDTargets(log logr.Logger, vpcCniConfigMap *v1.ConfigMap) (warmIPTarget int, minIPTarget int, warmPrefixTarget int) {
89-
warmIPTarget, minIPTarget, warmPrefixTarget = 0, 0, 0
90-
94+
// ParseWinIPTargetConfigs parses Windows IP target configuration parameters in the amazon-vpc-cni ConfigMap
95+
// If all three config parameter values (warm-ip-target, min-ip-target, warm-prefix-target) are 0 or unset, or config map does not exist,
96+
// then default values for warm-ip-target and min-ip-target will be set.
97+
func ParseWinIPTargetConfigs(log logr.Logger, vpcCniConfigMap *v1.ConfigMap) (warmIPTarget int, minIPTarget int, warmPrefixTarget int, isPDEnabled bool) {
9198
if vpcCniConfigMap.Data == nil {
92-
return warmIPTarget, minIPTarget, warmPrefixTarget
99+
warmIPTarget = IPv4DefaultWinWarmIPTarget
100+
minIPTarget = IPv4DefaultWinMinIPTarget
101+
log.V(1).Info(
102+
"No ConfigMap data found, falling back to using default values",
103+
"minIPTarget", minIPTarget,
104+
"warmIPTarget", warmIPTarget,
105+
)
106+
return warmIPTarget, minIPTarget, 0, false
107+
}
108+
109+
isPDEnabled, err := strconv.ParseBool(vpcCniConfigMap.Data[EnableWindowsPrefixDelegationKey])
110+
if err != nil {
111+
log.V(1).Info("Could not parse prefix delegation flag from ConfigMap, falling back to using secondary IP mode")
112+
isPDEnabled = false
93113
}
94114

95115
warmIPTargetStr, foundWarmIP := vpcCniConfigMap.Data[WarmIPTarget]
@@ -105,36 +125,67 @@ func ParseWinPDTargets(log logr.Logger, vpcCniConfigMap *v1.ConfigMap) (warmIPTa
105125
warmPrefixTargetStr, foundWarmPrefix = vpcCniConfigMap.Data[WinWarmPrefixTarget]
106126
}
107127

108-
// If no configuration is found, return 0
109-
if !foundWarmIP && !foundMinIP && !foundWarmPrefix {
110-
return warmIPTarget, minIPTarget, warmPrefixTarget
111-
}
112-
128+
// If warm IP target config value is not found, or there is an error parsing it, the value will be set to zero
113129
if foundWarmIP {
114130
warmIPTargetInt, err := strconv.Atoi(warmIPTargetStr)
115131
if err != nil {
116-
log.Error(err, "failed to parse warm ip target", "warm ip target", warmIPTargetStr)
132+
log.V(1).Info("could not parse warm ip target, defaulting to zero", "warm ip target", warmIPTargetStr)
133+
warmIPTarget = 0
117134
} else {
118135
warmIPTarget = warmIPTargetInt
119136
}
137+
} else {
138+
log.V(1).Info("could not find warm ip target in ConfigMap, defaulting to zero")
139+
warmIPTarget = 0
120140
}
141+
142+
// If min IP target config value is not found, or there is an error parsing it, the value will be set to zero
121143
if foundMinIP {
122144
minIPTargetInt, err := strconv.Atoi(minIPTargetStr)
123145
if err != nil {
124-
log.Error(err, "failed to parse minimum ip target", "minimum ip target", minIPTargetStr)
146+
log.V(1).Info("could not parse minimum ip target, defaulting to zero", "minimum ip target", minIPTargetStr)
147+
minIPTarget = 0
125148
} else {
126149
minIPTarget = minIPTargetInt
127150
}
151+
} else {
152+
log.V(1).Info("could not find minimum ip target in ConfigMap, defaulting to zero")
153+
minIPTarget = 0
128154
}
129-
if foundWarmPrefix {
155+
156+
// If PD is enabled and warm prefix target config value is not found, or there is an error parsing it, the value will be set to zero
157+
if !isPDEnabled && foundWarmPrefix {
158+
log.V(1).Info("warm prefix configuration not supported in secondary IP mode, will ignore warm prefix configuration")
159+
warmPrefixTarget = 0
160+
} else if isPDEnabled && foundWarmPrefix {
130161
warmPrefixTargetInt, err := strconv.Atoi(warmPrefixTargetStr)
131162
if err != nil {
132-
log.Error(err, "failed to parse warm prefix target", "warm prefix target", warmPrefixTargetStr)
163+
log.Error(err, "failed to parse warm prefix target, defaulting to zero", "warm prefix target", warmPrefixTargetStr)
164+
warmPrefixTarget = 0
133165
} else {
134166
warmPrefixTarget = warmPrefixTargetInt
135167
}
168+
} else if isPDEnabled && !foundWarmPrefix {
169+
log.V(1).Info("could not find warm prefix target in ConfigMap, defaulting to zero")
170+
warmPrefixTarget = 0
171+
}
172+
173+
if warmIPTarget == 0 && minIPTarget == 0 {
174+
if isPDEnabled && warmPrefixTarget == 0 {
175+
minIPTarget = IPv4PDDefaultMinIPTargetSize
176+
warmIPTarget = IPv4PDDefaultWarmIPTargetSize
177+
} else if !isPDEnabled {
178+
minIPTarget = IPv4DefaultWinMinIPTarget
179+
warmIPTarget = IPv4DefaultWinWarmIPTarget
180+
}
181+
log.V(1).Info(
182+
"No valid configuration values for warm-ip-target, min-ip-target and warm-prefix-target found in ConfigMap, falling back to using default values",
183+
"minIPTarget", minIPTarget,
184+
"warmIPTarget", warmIPTarget,
185+
)
136186
}
137-
return warmIPTarget, minIPTarget, warmPrefixTarget
187+
188+
return warmIPTarget, minIPTarget, warmPrefixTarget, isPDEnabled
138189
}
139190

140191
// getDefaultResourceConfig returns the default Resource Configuration.
@@ -153,13 +204,15 @@ func getDefaultResourceConfig() map[string]ResourceConfig {
153204

154205
// Create default configuration for IPv4 Resource
155206
ipV4WarmPoolConfig := WarmPoolConfig{
156-
DesiredSize: IPv4DefaultWPSize,
157-
MaxDeviation: IPv4DefaultMaxDev,
158-
ReservedSize: IPv4DefaultResSize,
207+
DesiredSize: IPv4DefaultWinWarmIPTarget,
208+
WarmIPTarget: IPv4DefaultWinWarmIPTarget,
209+
MinIPTarget: IPv4DefaultWinMinIPTarget,
210+
MaxDeviation: IPv4DefaultWinMaxDev,
211+
ReservedSize: IPv4DefaultWinResSize,
159212
}
160213
ipV4Config := ResourceConfig{
161214
Name: ResourceNameIPAddress,
162-
WorkerCount: IPv4DefaultWorker,
215+
WorkerCount: IPv4DefaultWinWorkerCount,
163216
SupportedOS: map[string]bool{OSWindows: true, OSLinux: false},
164217
WarmPoolConfig: &ipV4WarmPoolConfig,
165218
}

0 commit comments

Comments
 (0)