scheduler: gpuSharedPod doesn't fit secondaryDeviceWellPlanned (#2454)

ZiMengSheng · wangjianyu.wjy · web-flow · commit ed5d78ab77f3 · 2025-05-22T10:04:53.000Z
Signed-off-by: wangjianyu.wjy &lt;wangjianyu.wjy@alibaba-inc.com&gt;
Co-authored-by: wangjianyu.wjy &lt;wangjianyu.wjy@alibaba-inc.com&gt;
diff --git a/pkg/scheduler/plugins/deviceshare/device_allocator.go b/pkg/scheduler/plugins/deviceshare/device_allocator.go
@@ -77,7 +77,7 @@ func (a *AutopilotAllocator) Prepare() *framework.Status {
 	}
 	state := a.state
 	nodeDevice := a.nodeDevice
-	requestsPerInstance, desiredCountPerDeviceType, status := a.calcRequestsAndCountByDeviceType(state.podRequests, nodeDevice, state.hints, state.primaryDeviceType)
+	requestsPerInstance, desiredCountPerDeviceType, status := a.calcRequestsAndCountByDeviceType(state.podRequests, nodeDevice, state.hints, state.primaryDeviceType, state.podFitsSecondaryDeviceWellPlanned)
 	if !status.IsSuccess() {
 		return status
 	}
@@ -169,8 +169,9 @@ func (a *AutopilotAllocator) filterNodeDevice(
 }
 
 func (a *AutopilotAllocator) calcRequestsAndCountByDeviceType(
-	podRequests map[schedulingv1alpha1.DeviceType]corev1.ResourceList,
-	nodeDevice *nodeDevice, hints apiext.DeviceAllocateHints, primaryDeviceType schedulingv1alpha1.DeviceType,
+	podRequests map[schedulingv1alpha1.DeviceType]corev1.ResourceList, nodeDevice *nodeDevice,
+	hints apiext.DeviceAllocateHints, primaryDeviceType schedulingv1alpha1.DeviceType,
+	podFitsSecondaryDeviceWellPlanned bool,
 ) (map[schedulingv1alpha1.DeviceType]corev1.ResourceList, map[schedulingv1alpha1.DeviceType]int, *framework.Status) {
 	requestPerInstance := map[schedulingv1alpha1.DeviceType]corev1.ResourceList{}
 	desiredCountPerDeviceType := map[schedulingv1alpha1.DeviceType]int{}
@@ -184,7 +185,7 @@ func (a *AutopilotAllocator) calcRequestsAndCountByDeviceType(
 			continue
 		}
 
-		if primaryDeviceType != "" && deviceType != primaryDeviceType && nodeDevice.secondaryDeviceWellPlanned && a.phaseBeingExecuted != schedulingphase.Reserve {
+		if primaryDeviceType != "" && deviceType != primaryDeviceType && podFitsSecondaryDeviceWellPlanned && nodeDevice.secondaryDeviceWellPlanned && a.phaseBeingExecuted != schedulingphase.Reserve {
 			continue
 		}
 
diff --git a/pkg/scheduler/plugins/deviceshare/plugin.go b/pkg/scheduler/plugins/deviceshare/plugin.go
@@ -78,17 +78,18 @@ type Plugin struct {
 }
 
 type preFilterState struct {
-	skip               bool
-	podRequests        map[schedulingv1alpha1.DeviceType]corev1.ResourceList
-	hints              apiext.DeviceAllocateHints
-	hintSelectors      map[schedulingv1alpha1.DeviceType][2]labels.Selector
-	hasSelectors       bool
-	jointAllocate      *apiext.DeviceJointAllocate
-	primaryDeviceType  schedulingv1alpha1.DeviceType
-	gpuRequirements    *GPURequirements
-	allocationResult   apiext.DeviceAllocations
-	preemptibleDevices map[string]map[schedulingv1alpha1.DeviceType]deviceResources
-	preemptibleInRRs   map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources
+	skip                              bool
+	podRequests                       map[schedulingv1alpha1.DeviceType]corev1.ResourceList
+	hints                             apiext.DeviceAllocateHints
+	hintSelectors                     map[schedulingv1alpha1.DeviceType][2]labels.Selector
+	hasSelectors                      bool
+	jointAllocate                     *apiext.DeviceJointAllocate
+	primaryDeviceType                 schedulingv1alpha1.DeviceType
+	podFitsSecondaryDeviceWellPlanned bool
+	gpuRequirements                   *GPURequirements
+	allocationResult                  apiext.DeviceAllocations
+	preemptibleDevices                map[string]map[schedulingv1alpha1.DeviceType]deviceResources
+	preemptibleInRRs                  map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources
 
 	hasReservationAffinity bool
 }
@@ -106,16 +107,17 @@ type GPURequirements struct {
 
 func (s *preFilterState) Clone() framework.StateData {
 	ns := &preFilterState{
-		skip:                   s.skip,
-		podRequests:            s.podRequests,
-		hints:                  s.hints,
-		hasSelectors:           s.hasSelectors,
-		gpuRequirements:        s.gpuRequirements,
-		hintSelectors:          s.hintSelectors,
-		jointAllocate:          s.jointAllocate,
-		primaryDeviceType:      s.primaryDeviceType,
-		allocationResult:       s.allocationResult,
-		hasReservationAffinity: s.hasReservationAffinity,
+		skip:                              s.skip,
+		podRequests:                       s.podRequests,
+		hints:                             s.hints,
+		hasSelectors:                      s.hasSelectors,
+		gpuRequirements:                   s.gpuRequirements,
+		hintSelectors:                     s.hintSelectors,
+		jointAllocate:                     s.jointAllocate,
+		primaryDeviceType:                 s.primaryDeviceType,
+		podFitsSecondaryDeviceWellPlanned: s.podFitsSecondaryDeviceWellPlanned,
+		allocationResult:                  s.allocationResult,
+		hasReservationAffinity:            s.hasReservationAffinity,
 	}
 
 	preemptibleDevices := map[string]map[schedulingv1alpha1.DeviceType]deviceResources{}
diff --git a/pkg/scheduler/plugins/deviceshare/plugin_test.go b/pkg/scheduler/plugins/deviceshare/plugin_test.go
@@ -638,8 +638,9 @@ func Test_Plugin_PreFilter(t *testing.T) {
 						apiext.ResourceGPUMemoryRatio: resource.MustParse("100"),
 					},
 				},
-				preemptibleDevices: map[string]map[schedulingv1alpha1.DeviceType]deviceResources{},
-				preemptibleInRRs:   map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources{},
+				podFitsSecondaryDeviceWellPlanned: true,
+				preemptibleDevices:                map[string]map[schedulingv1alpha1.DeviceType]deviceResources{},
+				preemptibleInRRs:                  map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources{},
 			},
 		},
 		{
@@ -677,8 +678,9 @@ func Test_Plugin_PreFilter(t *testing.T) {
 						apiext.ResourceGPUMemoryRatio: resource.MustParse("100"),
 					},
 				},
-				preemptibleDevices: map[string]map[schedulingv1alpha1.DeviceType]deviceResources{},
-				preemptibleInRRs:   map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources{},
+				podFitsSecondaryDeviceWellPlanned: true,
+				preemptibleDevices:                map[string]map[schedulingv1alpha1.DeviceType]deviceResources{},
+				preemptibleInRRs:                  map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources{},
 			},
 		},
 		{
@@ -802,8 +804,9 @@ func Test_Plugin_PreFilter(t *testing.T) {
 						apiext.ResourceGPUMemoryRatio: resource.MustParse("100"),
 					},
 				},
-				preemptibleDevices: map[string]map[schedulingv1alpha1.DeviceType]deviceResources{},
-				preemptibleInRRs:   map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources{},
+				podFitsSecondaryDeviceWellPlanned: true,
+				preemptibleDevices:                map[string]map[schedulingv1alpha1.DeviceType]deviceResources{},
+				preemptibleInRRs:                  map[string]map[types.UID]map[schedulingv1alpha1.DeviceType]deviceResources{},
 			},
 		},
 		{
@@ -873,6 +876,7 @@ func Test_Plugin_PreFilter(t *testing.T) {
 						apiext.ResourceRDMA: resource.MustParse("100"),
 					},
 				},
+				podFitsSecondaryDeviceWellPlanned: true,
 				gpuRequirements: &GPURequirements{
 					numberOfGPUs: 1,
 					requestsPerGPU: corev1.ResourceList{
diff --git a/pkg/scheduler/plugins/deviceshare/utils.go b/pkg/scheduler/plugins/deviceshare/utils.go
@@ -322,6 +322,7 @@ func preparePod(pod *corev1.Pod) (state *preFilterState, status *framework.Statu
 		if err != nil {
 			return nil, framework.NewStatus(framework.UnschedulableAndUnresolvable, err.Error())
 		}
+		state.podFitsSecondaryDeviceWellPlanned = state.gpuRequirements != nil && !state.gpuRequirements.gpuShared
 		reservationAffinity, err := reservationutil.GetRequiredReservationAffinity(pod)
 		if err != nil {
 			return nil, framework.NewStatus(framework.UnschedulableAndUnresolvable, err.Error())

Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ func (a AutopilotAllocator) Prepare() framework.Status {`
`77`	`77`	`}`
`78`	`78`	`state := a.state`
`79`	`79`	`nodeDevice := a.nodeDevice`
`80`		`- requestsPerInstance, desiredCountPerDeviceType, status := a.calcRequestsAndCountByDeviceType(state.podRequests, nodeDevice, state.hints, state.primaryDeviceType)`
	`80`	`+ requestsPerInstance, desiredCountPerDeviceType, status := a.calcRequestsAndCountByDeviceType(state.podRequests, nodeDevice, state.hints, state.primaryDeviceType, state.podFitsSecondaryDeviceWellPlanned)`
`81`	`81`	`if !status.IsSuccess() {`
`82`	`82`	`return status`
`83`	`83`	`}`
`@@ -169,8 +169,9 @@ func (a *AutopilotAllocator) filterNodeDevice(`
`169`	`169`	`}`
`170`	`170`
`171`	`171`	`func (a *AutopilotAllocator) calcRequestsAndCountByDeviceType(`
`172`		`- podRequests map[schedulingv1alpha1.DeviceType]corev1.ResourceList,`
`173`		`- nodeDevice *nodeDevice, hints apiext.DeviceAllocateHints, primaryDeviceType schedulingv1alpha1.DeviceType,`
	`172`	`+ podRequests map[schedulingv1alpha1.DeviceType]corev1.ResourceList, nodeDevice *nodeDevice,`
	`173`	`+ hints apiext.DeviceAllocateHints, primaryDeviceType schedulingv1alpha1.DeviceType,`
	`174`	`+ podFitsSecondaryDeviceWellPlanned bool,`
`174`	`175`	`) (map[schedulingv1alpha1.DeviceType]corev1.ResourceList, map[schedulingv1alpha1.DeviceType]int, *framework.Status) {`
`175`	`176`	`requestPerInstance := map[schedulingv1alpha1.DeviceType]corev1.ResourceList{}`
`176`	`177`	`desiredCountPerDeviceType := map[schedulingv1alpha1.DeviceType]int{}`
`@@ -184,7 +185,7 @@ func (a *AutopilotAllocator) calcRequestsAndCountByDeviceType(`
`184`	`185`	`continue`
`185`	`186`	`}`
`186`	`187`
`187`		`- if primaryDeviceType != "" && deviceType != primaryDeviceType && nodeDevice.secondaryDeviceWellPlanned && a.phaseBeingExecuted != schedulingphase.Reserve {`
	`188`	`+ if primaryDeviceType != "" && deviceType != primaryDeviceType && podFitsSecondaryDeviceWellPlanned && nodeDevice.secondaryDeviceWellPlanned && a.phaseBeingExecuted != schedulingphase.Reserve {`
`188`	`189`	`continue`
`189`	`190`	`}`
`190`	`191`
Original file line number	Diff line number	Diff line change
`@@ -322,6 +322,7 @@ func preparePod(pod corev1.Pod) (state preFilterState, status *framework.Statu`
`322`	`322`	`if err != nil {`
`323`	`323`	`return nil, framework.NewStatus(framework.UnschedulableAndUnresolvable, err.Error())`
`324`	`324`	`}`
	`325`	`+ state.podFitsSecondaryDeviceWellPlanned = state.gpuRequirements != nil && !state.gpuRequirements.gpuShared`
`325`	`326`	`reservationAffinity, err := reservationutil.GetRequiredReservationAffinity(pod)`
`326`	`327`	`if err != nil {`
`327`	`328`	`return nil, framework.NewStatus(framework.UnschedulableAndUnresolvable, err.Error())`