Skip to content

Commit 8f727a4

Browse files
AiRanthemAiRanthem
authored
feature: range reserveOrdinals for AdvancedStatefulSet (openkruise#1873)
* feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem <[email protected]> * feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem <[email protected]> * feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem <[email protected]> * feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem <[email protected]> * feature: range reserveOrdinals for AdvancedStatefulSet Signed-off-by: AiRanthem <[email protected]> --------- Signed-off-by: AiRanthem <[email protected]>
1 parent 4183fbc commit 8f727a4

18 files changed

+528
-369
lines changed

‎apis/apps/v1beta1/statefulset_types.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,11 @@ limitations under the License.
1717
package v1beta1
1818

1919
import (
20+
appspub "github.com/openkruise/kruise/apis/apps/pub"
2021
apps "k8s.io/api/apps/v1"
2122
v1 "k8s.io/api/core/v1"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2324
"k8s.io/apimachinery/pkg/util/intstr"
24-
25-
appspub "github.com/openkruise/kruise/apis/apps/pub"
2625
)
2726

2827
const (
@@ -275,7 +274,8 @@ type StatefulSetSpec struct {
275274
// Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3])
276275
// - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2.
277276
// Then controller will delete Pod-1 (existing Pods will be [0, 2])
278-
ReserveOrdinals []int `json:"reserveOrdinals,omitempty"`
277+
// You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5].
278+
ReserveOrdinals []intstr.IntOrString `json:"reserveOrdinals,omitempty"`
279279

280280
// Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
281281
Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"`

‎apis/apps/v1beta1/zz_generated.deepcopy.go

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/apps.kruise.io_statefulsets.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -652,8 +652,12 @@ spec:
652652
Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3])
653653
- If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2.
654654
Then controller will delete Pod-1 (existing Pods will be [0, 2])
655+
You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5].
655656
items:
656-
type: integer
657+
anyOf:
658+
- type: integer
659+
- type: string
660+
x-kubernetes-int-or-string: true
657661
type: array
658662
revisionHistoryLimit:
659663
description: |-

‎config/crd/bases/apps.kruise.io_uniteddeployments.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -275,8 +275,12 @@ spec:
275275
Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3])
276276
- If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2.
277277
Then controller will delete Pod-1 (existing Pods will be [0, 2])
278+
You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5].
278279
items:
279-
type: integer
280+
anyOf:
281+
- type: integer
282+
- type: string
283+
x-kubernetes-int-or-string: true
280284
type: array
281285
revisionHistoryLimit:
282286
description: |-

‎pkg/controller/persistentpodstate/persistent_pod_state_controller.go

+4-5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"strings"
2525

2626
"github.com/openkruise/kruise/pkg/util/configuration"
27+
"k8s.io/utils/ptr"
2728

2829
ctrlUtil "github.com/openkruise/kruise/pkg/controller/util"
2930

@@ -36,7 +37,6 @@ import (
3637
"k8s.io/client-go/util/retry"
3738
"k8s.io/klog/v2"
3839
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
39-
utilpointer "k8s.io/utils/pointer"
4040
ctrl "sigs.k8s.io/controller-runtime"
4141
"sigs.k8s.io/controller-runtime/pkg/client"
4242
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -146,7 +146,7 @@ type innerStatefulset struct {
146146
// replicas
147147
Replicas int32
148148
// kruise statefulset filed
149-
ReserveOrdinals []int
149+
ReserveOrdinals sets.Set[int]
150150
DeletionTimestamp *metav1.Time
151151
}
152152

@@ -354,11 +354,10 @@ func (r *ReconcilePersistentPodState) getPodState(pod *corev1.Pod, nodeTopologyK
354354
}
355355

356356
func isInStatefulSetReplicas(index int, sts *innerStatefulset) bool {
357-
reserveOrdinals := sets.NewInt(sts.ReserveOrdinals...)
358357
replicas := sets.NewInt()
359358
replicaIndex := 0
360359
for realReplicaCount := 0; realReplicaCount < int(sts.Replicas); replicaIndex++ {
361-
if reserveOrdinals.Has(replicaIndex) {
360+
if sts.ReserveOrdinals.Has(replicaIndex) {
362361
continue
363362
}
364363
realReplicaCount++
@@ -457,7 +456,7 @@ func newStatefulSetPersistentPodState(workload *controllerfinder.ScaleAndSelecto
457456
APIVersion: workload.APIVersion,
458457
Kind: workload.Kind,
459458
Name: workload.Name,
460-
Controller: utilpointer.BoolPtr(true),
459+
Controller: ptr.To(true),
461460
UID: workload.UID,
462461
},
463462
},

‎pkg/controller/persistentpodstate/persistent_pod_state_controller_test.go

+11-6
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ import (
2323
"testing"
2424
"time"
2525

26+
"k8s.io/apimachinery/pkg/util/intstr"
2627
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
28+
"k8s.io/utils/ptr"
2729

2830
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
2931
appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1"
@@ -36,7 +38,6 @@ import (
3638
"k8s.io/apimachinery/pkg/runtime"
3739
"k8s.io/apimachinery/pkg/types"
3840
"k8s.io/kubernetes/pkg/apis/apps"
39-
"k8s.io/utils/pointer"
4041
"sigs.k8s.io/controller-runtime/pkg/client"
4142
"sigs.k8s.io/controller-runtime/pkg/client/fake"
4243
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -56,7 +57,7 @@ var (
5657
UID: "012d18d5-5eb9-449d-b670-3da8fec8852f",
5758
},
5859
Spec: appsv1beta1.StatefulSetSpec{
59-
Replicas: pointer.Int32Ptr(10),
60+
Replicas: ptr.To[int32](10),
6061
Template: corev1.PodTemplateSpec{
6162
ObjectMeta: metav1.ObjectMeta{
6263
Annotations: map[string]string{},
@@ -70,7 +71,7 @@ var (
7071
Namespace: "ns-test",
7172
OwnerReferences: []metav1.OwnerReference{
7273
{
73-
Controller: pointer.BoolPtr(true),
74+
Controller: ptr.To(true),
7475
},
7576
},
7677
Annotations: map[string]string{
@@ -222,7 +223,7 @@ func TestReconcilePersistentPodState(t *testing.T) {
222223
name: "kruise statefulset, scale down replicas 10->8, 1 pod deleted, 1 pod running",
223224
getSts: func() (*apps.StatefulSet, *appsv1beta1.StatefulSet) {
224225
kruise := kruiseStsDemo.DeepCopy()
225-
kruise.Spec.Replicas = pointer.Int32Ptr(8)
226+
kruise.Spec.Replicas = ptr.To[int32](8)
226227
return nil, kruise
227228
},
228229
getPods: func() []*corev1.Pod {
@@ -316,8 +317,12 @@ func TestReconcilePersistentPodState(t *testing.T) {
316317
name: "kruise reserveOrigin statefulset, scale down replicas 10->8, 1 pod deleted, 1 pod running",
317318
getSts: func() (*apps.StatefulSet, *appsv1beta1.StatefulSet) {
318319
kruise := kruiseStsDemo.DeepCopy()
319-
kruise.Spec.Replicas = pointer.Int32Ptr(8)
320-
kruise.Spec.ReserveOrdinals = []int{0, 3, 7}
320+
kruise.Spec.Replicas = ptr.To[int32](8)
321+
kruise.Spec.ReserveOrdinals = []intstr.IntOrString{
322+
intstr.FromInt32(0),
323+
intstr.FromInt32(3),
324+
intstr.FromInt32(7),
325+
}
321326
return nil, kruise
322327
},
323328
getPods: func() []*corev1.Pod {

‎pkg/controller/statefulset/stateful_pod_control_test.go

+69-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
/*
22
Copyright 2019 The Kruise Authors.
3-
Copyright 2016 The Kubernetes Authors.
43
54
Licensed under the Apache License, Version 2.0 (the "License");
65
you may not use this file except in compliance with the License.
@@ -34,6 +33,7 @@ import (
3433
"k8s.io/apimachinery/pkg/labels"
3534
"k8s.io/apimachinery/pkg/runtime"
3635
"k8s.io/apimachinery/pkg/types"
36+
"k8s.io/apimachinery/pkg/util/intstr"
3737
"k8s.io/client-go/kubernetes/fake"
3838
corelisters "k8s.io/client-go/listers/core/v1"
3939
storagelisters "k8s.io/client-go/listers/storage/v1"
@@ -1045,7 +1045,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
10451045
WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
10461046
WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType,
10471047
}
1048-
set.Spec.ReserveOrdinals = []int{2, 4}
1048+
set.Spec.ReserveOrdinals = []intstr.IntOrString{
1049+
intstr.FromInt32(2),
1050+
intstr.FromInt32(4),
1051+
}
10491052
return set
10501053
},
10511054
getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
@@ -1083,7 +1086,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
10831086
WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
10841087
WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
10851088
}
1086-
set.Spec.ReserveOrdinals = []int{2, 4}
1089+
set.Spec.ReserveOrdinals = []intstr.IntOrString{
1090+
intstr.FromInt32(2),
1091+
intstr.FromInt32(4),
1092+
}
10871093
return set
10881094
},
10891095
getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
@@ -1121,7 +1127,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
11211127
WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
11221128
WhenScaled: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
11231129
}
1124-
set.Spec.ReserveOrdinals = []int{2, 4}
1130+
set.Spec.ReserveOrdinals = []intstr.IntOrString{
1131+
intstr.FromInt32(2),
1132+
intstr.FromInt32(4),
1133+
}
11251134
return set
11261135
},
11271136
getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
@@ -1170,7 +1179,10 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
11701179
WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
11711180
WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType,
11721181
}
1173-
set.Spec.ReserveOrdinals = []int{2, 4}
1182+
set.Spec.ReserveOrdinals = []intstr.IntOrString{
1183+
intstr.FromInt32(2),
1184+
intstr.FromInt32(4),
1185+
}
11741186
return set
11751187
},
11761188
getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
@@ -1211,6 +1223,58 @@ func TestUpdatePodClaimForRetentionPolicy(t *testing.T) {
12111223
}
12121224
},
12131225
},
1226+
{
1227+
name: "reserveOrdinals is [1,3-5], scaleDown=true, whenScaled=Retain, whenDeleted=Delete",
1228+
getStatefulSet: func() *appsv1beta1.StatefulSet {
1229+
set := newStatefulSet(3)
1230+
set.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{
1231+
WhenDeleted: appsv1beta1.DeletePersistentVolumeClaimRetentionPolicyType,
1232+
WhenScaled: appsv1beta1.RetainPersistentVolumeClaimRetentionPolicyType,
1233+
}
1234+
set.Spec.ReserveOrdinals = []intstr.IntOrString{
1235+
intstr.FromInt32(1),
1236+
intstr.FromString("3-5"),
1237+
}
1238+
return set
1239+
},
1240+
getPods: func(set *appsv1beta1.StatefulSet) []*v1.Pod {
1241+
setClone := set.DeepCopy()
1242+
setClone.Spec.Replicas = utilpointer.Int32(5)
1243+
startOrdinal, endOrdinal, reserveOrdinals := getStatefulSetReplicasRange(setClone)
1244+
pods := make([]*v1.Pod, 0)
1245+
expectIndex := []int{0, 2, 6, 7, 8}
1246+
currentIndex := make([]int, 0)
1247+
for i := startOrdinal; i < endOrdinal; i++ {
1248+
if reserveOrdinals.Has(i) {
1249+
continue
1250+
}
1251+
currentIndex = append(currentIndex, i)
1252+
pods = append(pods, newStatefulSetPod(set, i))
1253+
}
1254+
if !reflect.DeepEqual(expectIndex, currentIndex) {
1255+
t.Fatalf("expect(%v), but get(%v)", expectIndex, currentIndex)
1256+
}
1257+
return pods
1258+
},
1259+
expectPvcOwnerRef: func(pvcName string) metav1.OwnerReference {
1260+
sIndex1 := strings.Index(pvcName, "-") + 1
1261+
podName := pvcName[sIndex1:]
1262+
sIndex2 := strings.LastIndex(pvcName, "-") + 1
1263+
index, _ := strconv.Atoi(pvcName[sIndex2:])
1264+
if index < 9 {
1265+
return metav1.OwnerReference{
1266+
APIVersion: "apps.kruise.io/v1beta1",
1267+
Kind: "StatefulSet",
1268+
Name: "foo",
1269+
}
1270+
}
1271+
return metav1.OwnerReference{
1272+
APIVersion: "v1",
1273+
Kind: "Pod",
1274+
Name: podName,
1275+
}
1276+
},
1277+
},
12141278
}
12151279

12161280
for _, cs := range cases {

‎pkg/controller/statefulset/stateful_set_control_test.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
/*
22
Copyright 2019 The Kruise Authors.
3-
Copyright 2016 The Kubernetes Authors.
43
54
Licensed under the Apache License, Version 2.0 (the "License");
65
you may not use this file except in compliance with the License.
@@ -4651,7 +4650,8 @@ func TestScaleUpWithMaxUnavailable(t *testing.T) {
46514650
}
46524651

46534652
func isOrHasInternalError(err error) bool {
4654-
agg, ok := err.(utilerrors.Aggregate)
4653+
var agg utilerrors.Aggregate
4654+
ok := errors.As(err, &agg)
46554655
return !ok && !apierrors.IsInternalError(err) || ok && len(agg.Errors()) > 0 && !apierrors.IsInternalError(agg.Errors()[0])
46564656
}
46574657

@@ -4662,10 +4662,12 @@ func emptyInvariants(set *appsv1beta1.StatefulSet, om *fakeObjectManager) error
46624662
func TestStatefulSetControlWithStartOrdinal(t *testing.T) {
46634663
defer utilfeature.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StatefulSetStartOrdinal, true)()
46644664

4665-
simpleSetFn := func(replicas, startOrdinal int, reservedIds ...int) *appsv1beta1.StatefulSet {
4665+
simpleSetFn := func(replicas, startOrdinal int, reservedIds ...int32) *appsv1beta1.StatefulSet {
46664666
statefulSet := newStatefulSet(replicas)
46674667
statefulSet.Spec.Ordinals = &appsv1beta1.StatefulSetOrdinals{Start: int32(startOrdinal)}
4668-
statefulSet.Spec.ReserveOrdinals = append([]int{}, reservedIds...)
4668+
for _, id := range reservedIds {
4669+
statefulSet.Spec.ReserveOrdinals = append(statefulSet.Spec.ReserveOrdinals, intstr.FromInt32(id))
4670+
}
46694671
return statefulSet
46704672
}
46714673

‎pkg/controller/statefulset/stateful_set_status_updater_test.go

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
/*
22
Copyright 2019 The Kruise Authors.
3-
Copyright 2017 The Kubernetes Authors.
43
54
Licensed under the Apache License, Version 2.0 (the "License");
65
you may not use this file except in compliance with the License.

‎pkg/controller/statefulset/stateful_set_utils.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"strconv"
2727
"time"
2828

29+
apiutil "github.com/openkruise/kruise/pkg/util/api"
2930
apps "k8s.io/api/apps/v1"
3031
v1 "k8s.io/api/core/v1"
3132
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -92,7 +93,7 @@ func podInOrdinalRange(pod *v1.Pod, set *appsv1beta1.StatefulSet) bool {
9293
return podInOrdinalRangeWithParams(pod, startOrdinal, endOrdinal, reserveOrdinals)
9394
}
9495

95-
func podInOrdinalRangeWithParams(pod *v1.Pod, startOrdinal, endOrdinal int, reserveOrdinals sets.Int) bool {
96+
func podInOrdinalRangeWithParams(pod *v1.Pod, startOrdinal, endOrdinal int, reserveOrdinals sets.Set[int]) bool {
9697
ordinal := getOrdinal(pod)
9798
return ordinal >= startOrdinal && ordinal < endOrdinal &&
9899
!reserveOrdinals.Has(ordinal)
@@ -791,8 +792,8 @@ func decreaseAndCheckMaxUnavailable(maxUnavailable *int) bool {
791792
// result is startOrdinal 2(inclusive), endOrdinal 7(exclusive), reserveOrdinals = {1, 3}
792793
// replicas[endOrdinal - startOrdinal] stores [replica-2, nil(reserveOrdinal 3), replica-4, replica-5, replica-6]
793794
// todo: maybe we should remove ineffective reserveOrdinals in webhook, reserveOrdinals = {3}
794-
func getStatefulSetReplicasRange(set *appsv1beta1.StatefulSet) (int, int, sets.Int) {
795-
reserveOrdinals := sets.NewInt(set.Spec.ReserveOrdinals...)
795+
func getStatefulSetReplicasRange(set *appsv1beta1.StatefulSet) (int, int, sets.Set[int]) {
796+
reserveOrdinals := apiutil.GetReserveOrdinalIntSet(set.Spec.ReserveOrdinals)
796797
replicaMaxOrdinal := getStartOrdinal(set)
797798
for realReplicaCount := 0; realReplicaCount < int(*set.Spec.Replicas); replicaMaxOrdinal++ {
798799
if reserveOrdinals.Has(replicaMaxOrdinal) {

0 commit comments

Comments
 (0)