Merge pull request #1614 from ingvagabund/nodeutilization-metrics-source

[nodeutilization]: allow to set a metrics source as a string so it can be later extended for exclusive configuration

Author: k8s-ci-robot

README.md

@@ -124,11 +124,22 @@ These are top level keys in the Descheduler Policy that you can use to configure
 | `maxNoOfPodsToEvictPerNode`        | `int`    | `nil`         | Maximum number of pods evicted from each node (summed through all strategies).                                             |
 | `maxNoOfPodsToEvictPerNamespace`   | `int`    | `nil`         | Maximum number of pods evicted from each namespace (summed through all strategies).                                        |
 | `maxNoOfPodsToEvictTotal`          | `int`    | `nil`         | Maximum number of pods evicted per rescheduling cycle (summed through all strategies).                                     |
-| `metricsCollector`                 | `object` | `nil`         | Configures collection of metrics for actual resource utilization.                                                          |
+| `metricsCollector` (deprecated)    | `object` | `nil`         | Configures collection of metrics for actual resource utilization.                                                          |
 | `metricsCollector.enabled`         | `bool`   | `false`       | Enables Kubernetes [Metrics Server](https://kubernetes-sigs.github.io/metrics-server/) collection.                         |
+| `metricsProviders`                 | `[]object` | `nil`       | Enables various metrics providers like Kubernetes [Metrics Server](https://kubernetes-sigs.github.io/metrics-server/)      |
 | `evictionFailureEventNotification` | `bool`   | `false`       | Enables eviction failure event notification.                                                                               |
 | `gracePeriodSeconds`               | `int`    | `0`           | The duration in seconds before the object should be deleted. The value zero indicates delete immediately.                  |
 
+The descheduler currently allows to configure a metric collection of Kubernetes Metrics through `metricsProviders` field.
+The previous way of setting `metricsCollector` field is deprecated. There is currently one source to configure:
+```
+metricsProviders:
+- source: KubernetesMetrics
+```
+The list can be extended with other metrics providers in the future.
+In general, each plugin can consume metrics from a different provider so multiple distinct providers can be configured in parallel.
+
+
 ### Evictor Plugin configuration (Default Evictor)
 
 The Default Evictor Plugin is used by default for filtering pods before processing them in an strategy plugin, or for applying a PreEvictionFilter of pods before eviction. You can also create your own Evictor Plugin or use the Default one provided by Descheduler.  Other uses for the Evictor plugin can be to sort, filter, validate or group pods by different criteria, and that's why this is handled by a plugin and not configured in the top level config.
@@ -163,8 +174,9 @@ maxNoOfPodsToEvictPerNode: 5000 # you don't need to set this, unlimited if not s
 maxNoOfPodsToEvictPerNamespace: 5000 # you don't need to set this, unlimited if not set
 maxNoOfPodsToEvictTotal: 5000 # you don't need to set this, unlimited if not set
 gracePeriodSeconds: 60 # you don't need to set this, 0 if not set
-metricsCollector:
-  enabled: true # you don't need to set this, metrics are not collected if not set
+# you don't need to set this, Kubernetes metrics are not collected if not set
+metricsProviders:
+- source: KubernetesMetrics
 profiles:
   - name: ProfileName
     pluginConfig:
@@ -288,9 +300,10 @@ A resource consumption above (resp. below) this window is considered as overutil
 This approach is chosen in order to maintain consistency with the kube-scheduler, which follows the same
 design for scheduling pods onto nodes. This means that resource usage as reported by Kubelet (or commands
 like `kubectl top`) may differ from the calculated consumption, due to these components reporting
-actual usage metrics. Metrics-based descheduling can be enabled by setting `metricsUtilization.metricsServer` field.
-In order to have the plugin consume the metrics the metric collector needs to be configured as well.
-See `metricsCollector` field at [Top Level configuration](#top-level-configuration) for available options.
+actual usage metrics. Metrics-based descheduling can be enabled by setting `metricsUtilization.metricsServer` field (deprecated)
+or `metricsUtilization.source` field to `KubernetesMetrics`.
+In order to have the plugin consume the metrics the metric provider needs to be configured as well.
+See `metricsProviders` field at [Top Level configuration](#top-level-configuration) for available options.
 
 **Parameters:**
 
@@ -303,7 +316,8 @@ See `metricsCollector` field at [Top Level configuration](#top-level-configurati
 |`evictionLimits`|object|
 |`evictableNamespaces`|(see [namespace filtering](#namespace-filtering))|
 |`metricsUtilization`|object|
-|`metricsUtilization.metricsServer`|bool|
+|`metricsUtilization.metricsServer` (deprecated)|bool|
+|`metricsUtilization.source`|string|
 
 
 **Example:**
@@ -325,7 +339,7 @@ profiles:
           "memory": 50
           "pods": 50
         metricsUtilization:
-          metricsServer: true
+          source: KubernetesMetrics
         evictionLimits:
           node: 5
     plugins:

charts/descheduler/templates/clusterrole.yaml

@@ -36,9 +36,13 @@ rules:
   resourceNames: ["{{ .Values.leaderElection.resourceName | default "descheduler" }}"]
   verbs: ["get", "patch", "delete"]
 {{- end }}
-{{- if and .Values.deschedulerPolicy .Values.deschedulerPolicy.metricsCollector .Values.deschedulerPolicy.metricsCollector.enabled }}
+{{- if and .Values.deschedulerPolicy }}
+{{- range .Values.deschedulerPolicy.metricsProviders }}
+{{- if and (hasKey . "source") (eq .source "KubernetesMetrics") }}
 - apiGroups: ["metrics.k8s.io"]
   resources: ["pods", "nodes"]
   verbs: ["get", "list"]
 {{- end }}
+{{- end }}
+{{- end }}
 {{- end -}}

charts/descheduler/values.yaml

@@ -96,8 +96,8 @@ deschedulerPolicy:
   # nodeSelector: "key1=value1,key2=value2"
   # maxNoOfPodsToEvictPerNode: 10
   # maxNoOfPodsToEvictPerNamespace: 10
-  # metricsCollector:
-  #   enabled: true
+  # metricsProviders:
+  # - source: KubernetesMetrics
   # ignorePvcPods: true
   # evictLocalStoragePods: true
   # evictDaemonSetPods: true

pkg/api/types.go

@@ -48,7 +48,11 @@ type DeschedulerPolicy struct {
 	EvictionFailureEventNotification *bool
 
 	// MetricsCollector configures collection of metrics about actual resource utilization
-	MetricsCollector MetricsCollector
+	// Deprecated. Use MetricsProviders field instead.
+	MetricsCollector *MetricsCollector
+
+	// MetricsProviders configure collection of metrics about actual resource utilization from various sources
+	MetricsProviders []MetricsProvider
 
 	// GracePeriodSeconds The duration in seconds before the object should be deleted. Value must be non-negative integer.
 	// The value zero indicates delete immediately. If this value is nil, the default grace period for the
@@ -105,12 +109,26 @@ type PluginSet struct {
 	Disabled []string
 }
 
+type MetricsSource string
+
+const (
+	// KubernetesMetrics enables metrics from a Kubernetes metrics server.
+	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
+	KubernetesMetrics MetricsSource = "KubernetesMetrics"
+)
+
 // MetricsCollector configures collection of metrics about actual resource utilization
 type MetricsCollector struct {
-	// Enabled metrics collection from kubernetes metrics.
-	// Later, the collection can be extended to other providers.
+	// Enabled metrics collection from Kubernetes metrics.
+	// Deprecated. Use MetricsProvider.Source field instead.
 	Enabled bool
 }
 
+// MetricsProvider configures collection of metrics about actual resource utilization from a given source
+type MetricsProvider struct {
+	// Source enables metrics from Kubernetes metrics server.
+	Source MetricsSource
+}
+
 // ReferencedResourceList is an adaption of v1.ResourceList with resources as references
 type ReferencedResourceList = map[v1.ResourceName]*resource.Quantity

pkg/api/v1alpha2/types.go

@@ -46,7 +46,11 @@ type DeschedulerPolicy struct {
 	EvictionFailureEventNotification *bool `json:"evictionFailureEventNotification,omitempty"`
 
 	// MetricsCollector configures collection of metrics for actual resource utilization
-	MetricsCollector MetricsCollector `json:"metricsCollector,omitempty"`
+	// Deprecated. Use MetricsProviders field instead.
+	MetricsCollector *MetricsCollector `json:"metricsCollector,omitempty"`
+
+	// MetricsProviders configure collection of metrics about actual resource utilization from various sources
+	MetricsProviders []MetricsProvider `json:"metricsProviders,omitempty"`
 
 	// GracePeriodSeconds The duration in seconds before the object should be deleted. Value must be non-negative integer.
 	// The value zero indicates delete immediately. If this value is nil, the default grace period for the
@@ -80,9 +84,23 @@ type PluginSet struct {
 	Disabled []string `json:"disabled"`
 }
 
+type MetricsSource string
+
+const (
+	// KubernetesMetrics enables metrics from a Kubernetes metrics server.
+	// Please see https://kubernetes-sigs.github.io/metrics-server/ for more.
+	KubernetesMetrics MetricsSource = "KubernetesMetrics"
+)
+
 // MetricsCollector configures collection of metrics about actual resource utilization
 type MetricsCollector struct {
-	// Enabled metrics collection from kubernetes metrics.
-	// Later, the collection can be extended to other providers.
+	// Enabled metrics collection from Kubernetes metrics server.
+	// Deprecated. Use MetricsProvider.Source field instead.
 	Enabled bool `json:"enabled,omitempty"`
 }
+
+// MetricsProvider configures collection of metrics about actual resource utilization from a given source
+type MetricsProvider struct {
+	// Source enables metrics from Kubernetes metrics server.
+	Source MetricsSource `json:"source,omitempty"`
+}

pkg/api/v1alpha2/zz_generated.conversion.go

@@ -166,7 +166,7 @@ func newDescheduler(ctx context.Context, rs *options.DeschedulerServer, deschedu
 	}
 
 	var metricsCollector *metricscollector.MetricsCollector
-	if deschedulerPolicy.MetricsCollector.Enabled {
+	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || (len(deschedulerPolicy.MetricsProviders) > 0 && deschedulerPolicy.MetricsProviders[0].Source == api.KubernetesMetrics) {
 		nodeSelector := labels.Everything()
 		if deschedulerPolicy.NodeSelector != nil {
 			sel, err := labels.Parse(*deschedulerPolicy.NodeSelector)
@@ -332,7 +332,7 @@ func Run(ctx context.Context, rs *options.DeschedulerServer) error {
 		return err
 	}
 
-	if deschedulerPolicy.MetricsCollector.Enabled {
+	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || (len(deschedulerPolicy.MetricsProviders) > 0 && deschedulerPolicy.MetricsProviders[0].Source == api.KubernetesMetrics) {
 		metricsClient, err := client.CreateMetricsClient(clientConnection, "descheduler")
 		if err != nil {
 			return err
@@ -448,7 +448,7 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 	sharedInformerFactory.WaitForCacheSync(ctx.Done())
 	descheduler.podEvictor.WaitForEventHandlersSync(ctx)
 
-	if deschedulerPolicy.MetricsCollector.Enabled {
+	if (deschedulerPolicy.MetricsCollector != nil && deschedulerPolicy.MetricsCollector.Enabled) || (len(deschedulerPolicy.MetricsProviders) > 0 && deschedulerPolicy.MetricsProviders[0].Source == api.KubernetesMetrics) {
 		go func() {
 			klog.V(2).Infof("Starting metrics collector")
 			descheduler.metricsCollector.Run(ctx)

pkg/api/v1alpha2/zz_generated.deepcopy.go

@@ -136,6 +136,10 @@ func removeDuplicatesPolicy() *api.DeschedulerPolicy {
 }
 
 func lowNodeUtilizationPolicy(thresholds, targetThresholds api.ResourceThresholds, metricsEnabled bool) *api.DeschedulerPolicy {
+	var metricsSource api.MetricsSource = ""
+	if metricsEnabled {
+		metricsSource = api.KubernetesMetrics
+	}
 	return &api.DeschedulerPolicy{
 		Profiles: []api.DeschedulerProfile{
 			{
@@ -146,8 +150,8 @@ func lowNodeUtilizationPolicy(thresholds, targetThresholds api.ResourceThreshold
 						Args: &nodeutilization.LowNodeUtilizationArgs{
 							Thresholds:       thresholds,
 							TargetThresholds: targetThresholds,
-							MetricsUtilization: nodeutilization.MetricsUtilization{
-								MetricsServer: metricsEnabled,
+							MetricsUtilization: &nodeutilization.MetricsUtilization{
+								Source: metricsSource,
 							},
 						},
 					},
@@ -837,7 +841,7 @@ func TestLoadAwareDescheduling(t *testing.T) {
 		},
 		true, // enabled metrics utilization
 	)
-	policy.MetricsCollector.Enabled = true
+	policy.MetricsProviders = []api.MetricsProvider{{Source: api.KubernetesMetrics}}
 
 	ctxCancel, cancel := context.WithCancel(ctx)
 	_, descheduler, _ := initDescheduler(