Quota check preceding resource check

pkg/controller/clusterstate/api/node_info.go

@@ -43,8 +43,10 @@ type NodeInfo struct {
 
 	// The releasing resource on that node
 	Releasing *Resource
+
 	// The idle resource on that node
 	Idle *Resource
+
 	// The used resource on that node, including running and terminating
 	// pods
 	Used *Resource
@@ -104,7 +106,6 @@ func (ni *NodeInfo) Clone() *NodeInfo {
 func (ni *NodeInfo) SetNode(node *v1.Node) {
 	if ni.Node == nil {
 		ni.Idle = NewResource(node.Status.Allocatable)
-
 	}
 
 	ni.Name = node.Name
@@ -121,5 +122,4 @@ func (ni NodeInfo) String() string {
 
 	return fmt.Sprintf("Node (%s): idle <%v>, used <%v>, releasing <%v>%s",
 		ni.Name, ni.Idle, ni.Used, ni.Releasing, res)
-
 }

pkg/controller/quota/quota_manager_interface.go

@@ -21,7 +21,7 @@ import (
 )
 
 type QuotaManagerInterface interface {
-	Fits(aw *arbv1.AppWrapper, resources *clusterstateapi.Resource, proposedPremptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string)
+	Fits(aw *arbv1.AppWrapper, requestedResources *clusterstateapi.Resource, clusterResources *clusterstateapi.Resource, proposedPremptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string)
 	Release(aw *arbv1.AppWrapper) bool
 	GetValidQuotaLabels() []string
 }

pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go

@@ -32,6 +32,7 @@ import (
 	qmbackend "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota"
 	qmbackendutils "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/quota-forest/quota-manager/quota/utils"
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/util"
+	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejobresources/genericresource"
 	"k8s.io/client-go/rest"
 
 	"math"
@@ -205,7 +206,7 @@ func (qm *QuotaManager) loadDispatchedAWs(dispatchedAWDemands map[string]*cluste
 			}
 			aw.SetLabels(newLabels)
 
-			doesFit, preemptionIds, errorMessage := qm.Fits(aw, v, nil)
+			doesFit, preemptionIds, errorMessage := qm.Fits(aw, v, nil, nil)
 			if !doesFit {
 				klog.Errorf("[loadDispatchedAWs] Loading of AppWrapper %s/%s failed.",
 					aw.Namespace, aw.Name)
@@ -509,7 +510,7 @@ func (qm *QuotaManager) buildRequest(aw *arbv1.AppWrapper,
 
 	return consumerInfo, err
 }
-func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource,
+func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource, clusterResources *clusterstateapi.Resource,
 	proposedPreemptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string) {
 
 	// If a Quota Manager Backend instance does not exists then assume quota failed
@@ -555,7 +556,7 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi
 
 	consumerID := consumerInfo.GetID()
 	klog.V(4).Infof("[Fits] Sending quota allocation request: %#v ", consumerInfo)
-	allocResponse, err := qm.quotaManagerBackend.AllocateForest(QuotaManagerForestName, consumerID)
+	allocResponse, err := qm.quotaManagerBackend.TryAllocateForest(QuotaManagerForestName, consumerID)
 	if err != nil {
 		qm.removeConsumer(consumerID)
 		klog.Errorf("[Fits] Error allocating consumer: %s/%s, err=%#v.", aw.Namespace, aw.Name, err)
@@ -569,9 +570,47 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi
 		return doesFit, preemptIds, strings.TrimSpace(allocResponse.GetMessage())
 	}
 	preemptIds = qm.getAppWrappers(allocResponse.GetPreemptedIds())
+
+	// Update cluster resources in the even that preemption happens
+	// TODO: Potentially move this resource updated out to the calling function (would need to comeback again to undo the allocation
+	// if the resources are not enough after preemption)
+	if clusterResources != nil {
+		updatedResources := clusterResources
+
+		for _, appWrapper := range preemptIds {
+			updatedResources.Add(qm.getAggregatedResources(appWrapper))
+		}
+
+		// Check if job fits with the update resources after preempted AppWrappers are removed
+		if clusterResources != nil && !awResDemands.LessEqual(updatedResources) {
+			qm.quotaManagerBackend.UndoAllocateForest(QuotaManagerForestName, consumerID)
+			qm.removeConsumer(consumerID)
+			return false, preemptIds, fmt.Sprintf("[Fits] AppWrapper '%s/%s' does not fit in the cluster, even after borrowed quota is freed", aw.Namespace, aw.Name)
+		}
+	}
+
 	return doesFit, preemptIds, strings.TrimSpace(allocResponse.GetMessage())
 }
 
+func (qm *QuotaManager) getAggregatedResources(appWrapper *arbv1.AppWrapper) *clusterstateapi.Resource {
+	// After quota evaluation, a set of AppWrappers is returned for preemption. Before deciding to delete them,
+	// we need to make sure enough resources are free for the new AppWrapper after the preemptable list is deleted.
+	// For this we need to add back the requests consumed by the preemptable AppWrappers to the available resources
+	// in order to perform a correct resource check with updated values.
+	allocated := clusterstateapi.EmptyResource()
+
+	for _, genericItem := range appWrapper.Spec.AggrResources.GenericItems {
+		resources, err := genericresource.GetResources(&genericItem)
+		if err != nil {
+			klog.V(8).Infof("[GetAggregatedResources] Failure aggregating resources for %s/%s, err=%#v, genericItem=%#v",
+				appWrapper.Namespace, appWrapper.Name, err, genericItem)
+		}
+		allocated = allocated.Add(resources)
+	}
+
+	return allocated
+}
+
 func (qm *QuotaManager) getAppWrappers(preemptIds []string) []*arbv1.AppWrapper {
 	var aws []*arbv1.AppWrapper
 	if len(preemptIds) <= 0 {

pkg/quotaplugins/quota-forest/quota-manager/quota/core/quotanode.go

@@ -138,7 +138,7 @@ func (qn *QuotaNode) SlideDown() {
 func (qn *QuotaNode) SlideUp(c *Consumer, applyPriority bool, allocationRecovery *AllocationRecovery,
 	preemptedConsumers *[]string) bool {
 
-	if qn.isHard {
+	if qn.isHard && !qn.IsRoot() {
 		return false
 	}
 

pkg/quotaplugins/quota-forest/quota-manager/quota/quotamanager.go

@@ -494,12 +494,14 @@ func (m *Manager) AllocateForest(forestName string, consumerID string) (response
 
 // TryAllocateForest : allocate a consumer on a forest
 func (m *Manager) TryAllocateForest(forestName string, consumerID string) (response *core.AllocationResponse, err error) {
+	if m.mode != Normal {
+		response, err = m.AllocateForest(forestName, consumerID)
+		return response, err
+	}
+
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	if m.mode != Normal {
-		return nil, fmt.Errorf("manager is not in normal mode")
-	}
 	forestController, forestConsumer, err := m.preAllocateForest(forestName, consumerID)
 	if err == nil && forestController.IsConsumerAllocated(consumerID) {
 		err = fmt.Errorf("consumer %s already allocated on forest %s", consumerID, forestName)

pkg/quotaplugins/quota-forest/quota-manager/quota/utils/defaults.go

@@ -21,7 +21,7 @@ var (
 	DefaultTreeName string = "default"
 
 	// DefaultResourceNames : the default resource names
-	DefaultResourceNames []string = []string{"cpu", "memory"}
+	DefaultResourceNames []string = []string{"cpu", "memory", "nvidia.com/gpu"}
 
 	// DefaultTreeKind : the default kind attribute of the tree
 	DefaultTreeKind string = "QuotaTree"

pkg/quotaplugins/quota-simple-rest/quota_rest_manager.go

@@ -257,7 +257,7 @@ func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) []QuotaGroup {
 	return groups
 }
 
-func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource,
+func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource, clusterResources *clusterstateapi.Resource,
 	proposedPreemptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string) {
 
 	// Handle uninitialized quota manager

test/e2e-kuttl/quota-forest/09-assert.yaml

@@ -1,4 +1,4 @@
-#Verify AppWrappers finished successfully
+# Verify AppWrappers finished successfully
 ---
 apiVersion: mcad.ibm.com/v1beta1
 kind: AppWrapper