scheduler: add ElasticQuota arg to control preemption of default quota (#2413) (#2449)

Signed-off-by: Lennon Chin <[email protected]>

Author: LennonChin

pkg/scheduler/apis/config/types.go

@@ -215,6 +215,9 @@ type ElasticQuotaArgs struct {
 	// EnableRuntimeQuota if false, use max instead of runtime for all checks.
 	EnableRuntimeQuota bool
 
+	// DisableDefaultQuotaPreemption if true, will not preempt pods in default quota.
+	DisableDefaultQuotaPreemption bool
+
 	// HookPlugins is expected to be configured with enabled hook plugins
 	HookPlugins []HookPluginConf
 }

pkg/scheduler/apis/config/v1/defaults.go

@@ -69,9 +69,10 @@ var (
 
 	defaultQuotaGroupNamespace = "koordinator-system"
 
-	defaultMonitorAllQuotas       = pointer.Bool(false)
-	defaultEnableCheckParentQuota = pointer.Bool(false)
-	defaultEnableRuntimeQuota     = pointer.Bool(true)
+	defaultMonitorAllQuotas              = pointer.Bool(false)
+	defaultEnableCheckParentQuota        = pointer.Bool(false)
+	defaultEnableRuntimeQuota            = pointer.Bool(true)
+	defaultDisableDefaultQuotaPreemption = pointer.Bool(true)
 
 	defaultTimeout           = 600 * time.Second
 	defaultControllerWorkers = 1
@@ -190,6 +191,9 @@ func SetDefaults_ElasticQuotaArgs(obj *ElasticQuotaArgs) {
 	if obj.EnableRuntimeQuota == nil {
 		obj.EnableRuntimeQuota = defaultEnableRuntimeQuota
 	}
+	if obj.DisableDefaultQuotaPreemption == nil {
+		obj.DisableDefaultQuotaPreemption = defaultDisableDefaultQuotaPreemption
+	}
 }
 
 func SetDefaults_CoschedulingArgs(obj *CoschedulingArgs) {

pkg/scheduler/apis/config/v1/types.go

@@ -209,6 +209,9 @@ type ElasticQuotaArgs struct {
 	// EnableRuntimeQuota if false, use max instead of runtime for all checks.
 	EnableRuntimeQuota *bool `json:"enableRuntimeQuota,omitempty"`
 
+	// DisableDefaultQuotaPreemption if true, will not preempt pods in default quota.
+	DisableDefaultQuotaPreemption *bool `json:"disableDefaultQuotaPreemption,omitempty"`
+
 	// HookPlugins is expected to be configured with enabled hook plugins
 	HookPlugins []HookPluginConf `json:"hookPlugins,omitempty"`
 }

pkg/scheduler/apis/config/v1/zz_generated.conversion.go

@@ -69,9 +69,10 @@ var (
 
 	defaultQuotaGroupNamespace = "koordinator-system"
 
-	defaultMonitorAllQuotas       = pointer.Bool(false)
-	defaultEnableCheckParentQuota = pointer.Bool(false)
-	defaultEnableRuntimeQuota     = pointer.Bool(true)
+	defaultMonitorAllQuotas              = pointer.Bool(false)
+	defaultEnableCheckParentQuota        = pointer.Bool(false)
+	defaultEnableRuntimeQuota            = pointer.Bool(true)
+	defaultDisableDefaultQuotaPreemption = pointer.Bool(true)
 
 	defaultTimeout           = 600 * time.Second
 	defaultControllerWorkers = 1
@@ -190,6 +191,9 @@ func SetDefaults_ElasticQuotaArgs(obj *ElasticQuotaArgs) {
 	if obj.EnableRuntimeQuota == nil {
 		obj.EnableRuntimeQuota = defaultEnableRuntimeQuota
 	}
+	if obj.DisableDefaultQuotaPreemption == nil {
+		obj.DisableDefaultQuotaPreemption = defaultDisableDefaultQuotaPreemption
+	}
 }
 
 func SetDefaults_CoschedulingArgs(obj *CoschedulingArgs) {

pkg/scheduler/apis/config/v1/zz_generated.deepcopy.go

@@ -209,6 +209,9 @@ type ElasticQuotaArgs struct {
 	// EnableRuntimeQuota if false, use max instead of runtime for all checks.
 	EnableRuntimeQuota *bool `json:"enableRuntimeQuota,omitempty"`
 
+	// DisableDefaultQuotaPreemption if true, will not preempt pods in default quota.
+	DisableDefaultQuotaPreemption *bool `json:"disableDefaultQuotaPreemption,omitempty"`
+
 	// HookPlugins is expected to be configured with enabled hook plugins
 	HookPlugins []HookPluginConf `json:"hookPlugins,omitempty"`
 }

pkg/scheduler/apis/config/v1beta3/defaults.go

@@ -290,5 +290,15 @@ func (g *Plugin) canPreempt(pod, victim *corev1.Pod) bool {
 	podQuotaName := g.getPodAssociateQuotaName(pod)
 	vicQuotaName := g.getPodAssociateQuotaName(victim)
 
+	// The quota of pods that do not actively set through annotation will be recognized as the default quota.
+	// In some scenarios, many important pods are not scheduled by Koordinator, and their quotas are
+	// also recognized as default quota, this will cause the eviction operation to identify these important
+	// pods as victim pods, which is risky. When DisableDefaultQuotaPreemption is set to true, these pods can
+	// be avoided from being evicted.
+	if g.pluginArgs.DisableDefaultQuotaPreemption &&
+		extension.DefaultQuotaName == vicQuotaName {
+		return false
+	}
+
 	return podPri > vicPri && podQuotaName == vicQuotaName
 }