Add Windows secondary IP mode configurable options for managing IP address allocation

controllers/core/configmap_controller.go

@@ -46,8 +46,8 @@ type ConfigMapReconciler struct {
 	Condition                         condition.Conditions
 	curWinIPAMEnabledCond             bool
 	curWinPrefixDelegationEnabledCond bool
-	curWinPDWarmIPTarget              int
-	curWinPDMinIPTarget               int
+	curWinWarmIPTarget                int
+	curWinMinIPTarget                 int
 	curWinPDWarmPrefixTarget          int
 	Context                           context.Context
 }
@@ -116,21 +116,33 @@ func (r *ConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 		isPrefixFlagUpdated = true
 	}
 
-	// Check if configurations for Windows prefix delegation have changed
-	var isPDConfigUpdated bool
-	warmIPTarget, minIPTarget, warmPrefixTarget := config.ParseWinPDTargets(r.Log, configmap)
-	if r.curWinPDWarmIPTarget != warmIPTarget || r.curWinPDMinIPTarget != minIPTarget || r.curWinPDWarmPrefixTarget != warmPrefixTarget {
-		r.curWinPDWarmIPTarget = warmIPTarget
-		r.curWinPDMinIPTarget = minIPTarget
+	// Check if Windows IP target configurations in ConfigMap have changed
+	var isWinIPConfigsUpdated bool
+
+	warmIPTarget, minIPTarget, warmPrefixTarget, isPDEnabled := config.ParseWinIPTargetConfigs(r.Log, configmap)
+	var winMinIPTargetUpdated = r.curWinMinIPTarget != minIPTarget
+	var winWarmIPTargetUpdated = r.curWinWarmIPTarget != warmIPTarget
+	var winPDWarmPrefixTargetUpdated = r.curWinPDWarmPrefixTarget != warmPrefixTarget
+	if winWarmIPTargetUpdated || winMinIPTargetUpdated {
+		r.curWinWarmIPTarget = warmIPTarget
+		r.curWinMinIPTarget = minIPTarget
+		isWinIPConfigsUpdated = true
+	}
+	if isPDEnabled && winPDWarmPrefixTargetUpdated {
 		r.curWinPDWarmPrefixTarget = warmPrefixTarget
-		logger.Info("updated PD configs from configmap", config.WarmIPTarget, r.curWinPDWarmIPTarget,
-			config.MinimumIPTarget, r.curWinPDMinIPTarget, config.WarmPrefixTarget, r.curWinPDWarmPrefixTarget)
-
-		isPDConfigUpdated = true
+		isWinIPConfigsUpdated = true
+	}
+	if isWinIPConfigsUpdated {
+		logger.Info(
+			"Detected update in Windows IP configuration parameter values in ConfigMap",
+			config.WinWarmIPTarget, r.curWinWarmIPTarget,
+			config.WinMinimumIPTarget, r.curWinMinIPTarget,
+			config.WinWarmPrefixTarget, r.curWinPDWarmPrefixTarget,
+			config.EnableWindowsPrefixDelegationKey, isPDEnabled,
+		)
 	}
 
-	// Flag is updated, update all nodes
-	if isIPAMFlagUpdated || isPrefixFlagUpdated || isPDConfigUpdated {
+	if isIPAMFlagUpdated || isPrefixFlagUpdated || isWinIPConfigsUpdated {
 		err := UpdateNodesOnConfigMapChanges(r.K8sAPI, r.NodeManager)
 		if err != nil {
 			// Error in updating nodes

controllers/core/configmap_controller_test.go

@@ -16,14 +16,9 @@ package controllers
 import (
 	"context"
 	"errors"
+	"strconv"
 	"testing"
 
-	mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
-	mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
-	mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
-	mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
-	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
-	cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
@@ -35,18 +30,36 @@ import (
 	fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	mock_condition "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/condition"
+	mock_k8s "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/k8s"
+	mock_node "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node"
+	mock_manager "github.com/aws/amazon-vpc-resource-controller-k8s/mocks/amazon-vcp-resource-controller-k8s/pkg/node/manager"
+	"github.com/aws/amazon-vpc-resource-controller-k8s/pkg/config"
+	cooldown "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/provider/branch/cooldown"
 )
 
 var (
 	mockConfigMap = &corev1.ConfigMap{
 		TypeMeta:   metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
-		Data:       map[string]string{config.EnableWindowsIPAMKey: "true", config.EnableWindowsPrefixDelegationKey: "true"},
+		Data: map[string]string{
+			config.EnableWindowsIPAMKey:             "true",
+			config.EnableWindowsPrefixDelegationKey: "false",
+			config.WinMinimumIPTarget:               strconv.Itoa(config.IPv4DefaultWinMinIPTarget),
+			config.WinWarmIPTarget:                  strconv.Itoa(config.IPv4DefaultWinWarmIPTarget),
+		},
 	}
 	mockConfigMapPD = &corev1.ConfigMap{
 		TypeMeta:   metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{Name: config.VpcCniConfigMapName, Namespace: config.KubeSystemNamespace},
-		Data:       map[string]string{config.EnableWindowsIPAMKey: "false", config.EnableWindowsPrefixDelegationKey: "true"},
+		Data: map[string]string{
+			config.EnableWindowsIPAMKey:             "true",
+			config.EnableWindowsPrefixDelegationKey: "true",
+			config.WinMinimumIPTarget:               strconv.Itoa(config.IPv4PDDefaultMinIPTargetSize),
+			config.WinWarmIPTarget:                  strconv.Itoa(config.IPv4PDDefaultWarmIPTargetSize),
+			config.WinWarmPrefixTarget:              strconv.Itoa(config.IPv4PDDefaultWarmPrefixTargetSize),
+		},
 	}
 	mockConfigMapReq = reconcile.Request{
 		NamespacedName: types.NamespacedName{
@@ -89,11 +102,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
 	return ConfigMapMock{
 		MockNodeManager: mockNodeManager,
 		ConfigMapReconciler: &ConfigMapReconciler{
-			Client:      client,
-			Log:         zap.New(),
-			NodeManager: mockNodeManager,
-			K8sAPI:      mockK8sWrapper,
-			Condition:   mockCondition,
+			Client:             client,
+			Log:                zap.New(),
+			NodeManager:        mockNodeManager,
+			K8sAPI:             mockK8sWrapper,
+			Condition:          mockCondition,
+			curWinMinIPTarget:  config.IPv4DefaultWinMinIPTarget,
+			curWinWarmIPTarget: config.IPv4DefaultWinWarmIPTarget,
 		},
 		MockNode:                   mockNode,
 		MockK8sAPI:                 mockK8sWrapper,
@@ -103,13 +118,13 @@ func NewConfigMapMock(ctrl *gomock.Controller, mockObjects ...client.Object) Con
 	}
 }
 
-func Test_Reconcile_ConfigMap_Updated(t *testing.T) {
+func Test_Reconcile_ConfigMap_Updated_Secondary_IP(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
 	mock := NewConfigMapMock(ctrl, mockConfigMap)
 	mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(true)
-	mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true)
+	mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
 	mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil)
 	mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true)
 	mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil)
@@ -120,14 +135,32 @@ func Test_Reconcile_ConfigMap_Updated(t *testing.T) {
 	res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq)
 	assert.NoError(t, err)
 	assert.Equal(t, res, reconcile.Result{})
+}
 
+func Test_Reconcile_ConfigMap_Updated_PD(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mock := NewConfigMapMock(ctrl, mockConfigMapPD)
+	mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(true)
+	mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(true)
+	mock.MockK8sAPI.EXPECT().ListNodes().Return(nodeList, nil)
+	mock.MockNodeManager.EXPECT().GetNode(mockNodeName).Return(mock.MockNode, true)
+	mock.MockNodeManager.EXPECT().UpdateNode(mockNodeName).Return(nil)
+
+	mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()
+
+	cooldown.InitCoolDownPeriod(mock.MockK8sAPI, zap.New(zap.UseDevMode(true)).WithName("cooldown"))
+	res, err := mock.ConfigMapReconciler.Reconcile(context.TODO(), mockConfigMapReq)
+	assert.NoError(t, err)
+	assert.Equal(t, res, reconcile.Result{})
 }
 
 func Test_Reconcile_ConfigMap_PD_Disabled_If_IPAM_Disabled(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
-	mock := NewConfigMapMock(ctrl, mockConfigMapPD)
+	mock := NewConfigMapMock(ctrl, mockConfigMap)
 	mock.MockCondition.EXPECT().IsWindowsIPAMEnabled().Return(false)
 	mock.MockCondition.EXPECT().IsWindowsPrefixDelegationEnabled().Return(false)
 	mock.MockK8sAPI.EXPECT().GetConfigMap(config.VpcCniConfigMapName, config.KubeSystemNamespace).Return(createCoolDownMockCM("30"), nil).AnyTimes()

docs/windows/secondary_ip_mode_config_options.md

@@ -0,0 +1,63 @@
+# Configuration options when using secondary IP addresses Windows
+
+We provide multiple configuration options that allow you to fine-tune the IP address allocation behavior on Windows
+nodes using the secondary IP address mode. These configuration options can be set in the `amazon-vpc-cni` ConfigMap in
+the `kube-system` namespace.
+
+- `windows-warm-ip-target` → The total number of IP addresses that should be allocated to each Windows node in excess of
+  the current need at any given time. The excess IPs can be used by newly launched pods, which aids in faster pod
+  startup times since there is no wait time for additional IP addresses to be allocated. The VPC Resource Controller
+  will attempt to ensure that this excess desired threshold is always met.
+
+  Defaults to 3 if unspecified or invalid. Must be greater than or equal to 1.
+
+  For example, if no pods were running on a given Windows node, and if you set `windows-warm-ip-target` to 5, the VPC
+  Resource Controller will aim to ensure that each Windows node always has at least 5 IP addresses in excess, ready for
+  use, allocated to its ENI. If 2 pods are scheduled on the node, the controller will allocate 2 additional IP addresses
+  to the ENI, maintaining the 5 warm IP address target.
+
+- `windows-minimum-ip-target` → Defaults to 3 if unspecified or invalid. The minimum number of IP addresses, both in use
+  by running pods and available as warm IPs, that should be allocated to each Windows node at any given time. The
+  controller will attempt to ensure that this minimum threshold is always met.
+
+  Defaults to 3 if unspecified or invalid. Must be greater than or equal to 0.
+
+  For example, if no pods were running on a given Windows node, and if you set `windows-minimum-ip-target` to 10, the
+  VPC Resource Controller will aim to ensure that the total number of IP addresses on the Windows node should be at
+  least 10. Therefore, before pods are scheduled, there should be at least 10 IP addresses available. If 5 pods are
+  scheduled on a given node, they will consume 5 of the 10 available IPs. The VPC Resource Controller will keep 5 the
+  remaining available IPs available in addition to the 5 already in use to meet the target of 10.
+
+### Considerations while using the above configuration options
+
+- These configuration options only apply when the VPC Resource Controller is operating in the secondary IP mode. They do
+  not affect the prefix delegation mode. More explicitly, if `enable-windows-prefix-delegation` is set to false, or is
+  not specified, then the VPC Resource Controller operates in secondary IP mode.
+- Setting either `windows-warm-ip-target` or `windows-minimum-ip-target` to a negative value will result in the
+  respective default value being used.
+- If the values of `windows-warm-ip-target` or `windows-minimum-ip-target` are set such that the maximum node IP
+  capacity would be exceeded, the controller will limit the allocation to the maximum capacity possible.
+- The `warm-prefix-target` configuration option will be ignored when using the secondary IP mode, as it only applies to
+  the prefix delegation mode.
+- If `windows-warm-ip-target` is set to 0, the system will implicitly set `windows-warm-ip-target` to 1. This is
+  because on-demand IP allocation whereby an IP is allocated on the Windows node as the pods are scheduled is currently
+  not supported. Implicitly Setting `windows-warm-ip-target` to 1 ensures the minimum acceptable non-zero value is set
+  since the `windows-warm-ip-target` should always be at least 1.
+- The configuration options `warm-ip-target` and `minimum-ip-target` are deprecated in favor of the new
+  options `windows-warm-ip-target` and `windows-minimum-ip-target`.
+
+### Examples
+
+| `windows-warm-ip-target` | `windows-minimum-ip-target` | Running Pods | Total Allocated IPs | Warm IPs |
+|--------------------------|-----------------------------|--------------|---------------------|----------|
+| 1                        | 0                           | 0            | 1                   | 1        |
+| 1                        | 0                           | 5            | 6                   | 1        |
+| 5                        | 0                           | 0            | 5                   | 5        |
+| 1                        | 1                           | 0            | 1                   | 1        |
+| 1                        | 1                           | 1            | 2                   | 1        |
+| 1                        | 3                           | 3            | 4                   | 1        |
+| 1                        | 3                           | 5            | 6                   | 1        |
+| 5                        | 10                          | 0            | 10                  | 10       |
+| 10                       | 10                          | 0            | 10                  | 10       |
+| 10                       | 10                          | 10           | 20                  | 10       |
+| 15                       | 10                          | 10           | 25                  | 15       |