[YUNIKORN-2854]Add queue maxRunningApps metrics (#1012)

kaichiachen · pbacsko · commit 4f2f1b3865a8 · 2025-02-18T15:23:52.000+01:00
Closes: #1012 Signed-off-by: Peter Bacsko <pbacsko@cloudera.com>
diff --git a/pkg/metrics/queue.go b/pkg/metrics/queue.go
@@ -44,10 +44,11 @@ const (
 	ContainerAllocated = "allocated"
 	ContainerRejected  = "rejected"
 
-	QueueGuaranteed = "guaranteed"
-	QueueMax        = "max"
-	QueuePending    = "pending"
-	QueuePreempting = "preempting"
+	QueueGuaranteed     = "guaranteed"
+	QueueMax            = "max"
+	QueuePending        = "pending"
+	QueuePreempting     = "preempting"
+	QueueMaxRunningApps = "maxRunningApps"
 )
 
 // QueueMetrics to declare queue metrics
@@ -99,15 +100,15 @@ func InitQueueMetrics(name string) *QueueMetrics {
 			Namespace:   Namespace,
 			Name:        "queue_resource",
 			ConstLabels: prometheus.Labels{"queue": name},
-			Help:        "Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`.",
+			Help:        "Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`, `maxRunningApps`.",
 		}, []string{"state", "resource"})
 
 	q.resourceMetricsSubsystem = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: Namespace,
 			Subsystem: replaceStr,
 			Name:      "queue_resource",
-			Help:      "Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`.",
+			Help:      "Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`, `maxRunningApps`.",
 		}, []string{"state", "resource"})
 
 	var queueMetricsList = []prometheus.Collector{
@@ -354,3 +355,7 @@ func (m *QueueMetrics) SetQueuePendingResourceMetrics(resourceName string, value
 func (m *QueueMetrics) SetQueuePreemptingResourceMetrics(resourceName string, value float64) {
 	m.setQueueResource(QueuePreempting, resourceName, value)
 }
+
+func (m *QueueMetrics) SetQueueMaxRunningAppsMetrics(value uint64) {
+	m.setQueueResource(QueueMaxRunningApps, "apps", float64(value))
+}
diff --git a/pkg/metrics/queue_test.go b/pkg/metrics/queue_test.go
@@ -246,6 +246,14 @@ func TestQueuePreemptingResourceMetrics(t *testing.T) {
 	verifyResourceMetrics(t, "preempting", "cpu")
 }
 
+func TestQueueMaxRunningAppsResourceMetrics(t *testing.T) {
+	qm = getQueueMetrics()
+	defer unregisterQueueMetrics()
+
+	qm.SetQueueMaxRunningAppsMetrics(1)
+	verifyResourceMetrics(t, "maxRunningApps", "apps")
+}
+
 func TestRemoveQueueMetrics(t *testing.T) {
 	testQueueName := "root.test"
 	qm = GetQueueMetrics(testQueueName)
diff --git a/pkg/scheduler/objects/queue.go b/pkg/scheduler/objects/queue.go
@@ -125,6 +125,7 @@ func NewConfiguredQueue(conf configs.QueueConfig, parent *Queue) (*Queue, error)
 	sq.parent = parent
 	sq.isManaged = true
 	sq.maxRunningApps = conf.MaxApplications
+	sq.updateMaxRunningAppsMetrics()
 
 	// update the properties
 	if err := sq.applyConf(conf); err != nil {
@@ -223,6 +224,7 @@ func (sq *Queue) applyTemplate(childTemplate *template.Template) {
 	// update metrics for guaranteed and max resource
 	sq.updateGuaranteedResourceMetrics()
 	sq.updateMaxResourceMetrics()
+	sq.updateMaxRunningAppsMetrics()
 }
 
 // getProperties returns a copy of the properties for this queue
@@ -366,6 +368,7 @@ func (sq *Queue) applyConf(conf configs.QueueConfig) error {
 			return err
 		}
 		sq.maxRunningApps = conf.MaxApplications
+		sq.updateMaxRunningAppsMetrics()
 	}
 
 	sq.properties = conf.Properties
@@ -462,6 +465,7 @@ func (sq *Queue) SetMaxRunningApps(maxApps uint64) {
 	sq.Lock()
 	defer sq.Unlock()
 	sq.maxRunningApps = maxApps
+	sq.updateMaxRunningAppsMetrics()
 }
 
 // setTemplate sets the template on the queue based on the config.
@@ -1705,6 +1709,10 @@ func (sq *Queue) updatePreemptingResourceMetrics() {
 	}
 }
 
+func (sq *Queue) updateMaxRunningAppsMetrics() {
+	metrics.GetQueueMetrics(sq.QueuePath).SetQueueMaxRunningAppsMetrics(sq.maxRunningApps)
+}
+
 func (sq *Queue) removeMetrics() {
 	metrics.RemoveQueueMetrics(sq.QueuePath)
 }
diff --git a/pkg/scheduler/objects/queue_test.go b/pkg/scheduler/objects/queue_test.go
@@ -298,9 +298,11 @@ func TestPendingCalc(t *testing.T) {
 	want := concatQueueResourceMetric(metrics, []string{`
 yunikorn_root_queue_resource{resource="memory",state="pending"} 100
 yunikorn_root_queue_resource{resource="vcores",state="pending"} 10
+yunikorn_root_queue_resource{resource="apps",state="maxRunningApps"} 0
 `, `
 yunikorn_root_leaf_queue_resource{resource="memory",state="pending"} 100
 yunikorn_root_leaf_queue_resource{resource="vcores",state="pending"} 10
+yunikorn_root_leaf_queue_resource{resource="apps",state="maxRunningApps"} 0
 `},
 	)
 	assert.NilError(t, promtu.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(want), metrics...), "unexpected metrics")
@@ -314,9 +316,11 @@ yunikorn_root_leaf_queue_resource{resource="vcores",state="pending"} 10
 	want = concatQueueResourceMetric(metrics, []string{`
 yunikorn_root_queue_resource{resource="memory",state="pending"} 0
 yunikorn_root_queue_resource{resource="vcores",state="pending"} 0
+yunikorn_root_queue_resource{resource="apps",state="maxRunningApps"} 0
 `, `
 yunikorn_root_leaf_queue_resource{resource="memory",state="pending"} 0
 yunikorn_root_leaf_queue_resource{resource="vcores",state="pending"} 0
+yunikorn_root_leaf_queue_resource{resource="apps",state="maxRunningApps"} 0
 `},
 	)
 	assert.NilError(t, promtu.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(want), metrics...), "unexpected metrics")
@@ -334,16 +338,18 @@ yunikorn_root_leaf_queue_resource{resource="vcores",state="pending"} 0
 	want = concatQueueResourceMetric(metrics, []string{`
 yunikorn_root_queue_resource{resource="memory",state="pending"} 0
 yunikorn_root_queue_resource{resource="vcores",state="pending"} 0
+yunikorn_root_queue_resource{resource="apps",state="maxRunningApps"} 0
 `, `
 yunikorn_root_leaf_queue_resource{resource="memory",state="pending"} 0
 yunikorn_root_leaf_queue_resource{resource="vcores",state="pending"} 0
+yunikorn_root_leaf_queue_resource{resource="apps",state="maxRunningApps"} 0
 `},
 	)
 	assert.NilError(t, promtu.GatherAndCompare(prometheus.DefaultGatherer, strings.NewReader(want), metrics...), "unexpected metrics")
 }
 
 const (
-	QueueResourceMetricHelp = "# HELP %v Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`."
+	QueueResourceMetricHelp = "# HELP %v Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`, `maxRunningApps`."
 	QueueResourceMetricType = "# TYPE %v gauge"
 )
 

Original file line number	Diff line number	Diff line change
`@@ -125,6 +125,7 @@ func NewConfiguredQueue(conf configs.QueueConfig, parent Queue) (Queue, error)`
`125`	`125`	`sq.parent = parent`
`126`	`126`	`sq.isManaged = true`
`127`	`127`	`sq.maxRunningApps = conf.MaxApplications`
	`128`	`+ sq.updateMaxRunningAppsMetrics()`
`128`	`129`
`129`	`130`	`// update the properties`
`130`	`131`	`if err := sq.applyConf(conf); err != nil {`
`@@ -223,6 +224,7 @@ func (sq Queue) applyTemplate(childTemplate template.Template) {`
`223`	`224`	`// update metrics for guaranteed and max resource`
`224`	`225`	`sq.updateGuaranteedResourceMetrics()`
`225`	`226`	`sq.updateMaxResourceMetrics()`
	`227`	`+ sq.updateMaxRunningAppsMetrics()`
`226`	`228`	`}`
`227`	`229`
`228`	`230`	`// getProperties returns a copy of the properties for this queue`
`@@ -366,6 +368,7 @@ func (sq *Queue) applyConf(conf configs.QueueConfig) error {`
`366`	`368`	`return err`
`367`	`369`	`}`
`368`	`370`	`sq.maxRunningApps = conf.MaxApplications`
	`371`	`+ sq.updateMaxRunningAppsMetrics()`
`369`	`372`	`}`
`370`	`373`
`371`	`374`	`sq.properties = conf.Properties`
`@@ -462,6 +465,7 @@ func (sq *Queue) SetMaxRunningApps(maxApps uint64) {`
`462`	`465`	`sq.Lock()`
`463`	`466`	`defer sq.Unlock()`
`464`	`467`	`sq.maxRunningApps = maxApps`
	`468`	`+ sq.updateMaxRunningAppsMetrics()`
`465`	`469`	`}`
`466`	`470`
`467`	`471`	`// setTemplate sets the template on the queue based on the config.`
`@@ -1705,6 +1709,10 @@ func (sq *Queue) updatePreemptingResourceMetrics() {`
`1705`	`1709`	`}`
`1706`	`1710`	`}`
`1707`	`1711`
	`1712`	`+func (sq *Queue) updateMaxRunningAppsMetrics() {`
	`1713`	`+ metrics.GetQueueMetrics(sq.QueuePath).SetQueueMaxRunningAppsMetrics(sq.maxRunningApps)`
	`1714`	`+}`
	`1715`	`+`
`1708`	`1716`	`func (sq *Queue) removeMetrics() {`
`1709`	`1717`	`metrics.RemoveQueueMetrics(sq.QueuePath)`
`1710`	`1718`	`}`