Add initial Kueue integration

Signed-off-by: ted chang <[email protected]>

Author: tedhtchang

api/lmes/v1alpha1/lmevaljob_types.go

@@ -236,6 +236,8 @@ type LMEvalJobSpec struct {
 	// Specify extra information for the lm-eval job's pod
 	// +optional
 	Pod *LMEvalPodSpec `json:"pod,omitempty"`
+	// This is for Kueue integration. Kueue requires this value to be false initially.
+	Suspend bool `json:"suspend,omitempty"`
 }
 
 // LMEvalJobStatus defines the observed state of LMEvalJob

api/lmes/v1alpha1/zz_generated.deepcopy.go

@@ -36,6 +36,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	lmesv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/lmes/v1alpha1"
 	tasv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/tas/v1alpha1"
@@ -89,12 +90,10 @@ func main() {
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		Port:                   9443,
+		Metrics:                server.Options{BindAddress: metricsAddr + ":9443"},
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "b7e9931f.trustyai.opendatahub.io",
-		Namespace:              "", // We are defining a cluster-scoped operator
 		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
 		// when the Manager ends. This requires the binary to immediately end when the
 		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly

api/tas/v1alpha1/zz_generated.deepcopy.go

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.16.3
   name: trustyaiservices.trustyai.opendatahub.io
 spec:
   group: trustyai.opendatahub.io
@@ -21,14 +20,19 @@ spec:
         description: TrustyAIService is the Schema for the trustyaiservices API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object

cmd/operator/main.go

@@ -2,13 +2,16 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: manager-role
 rules:
 - apiGroups:
   - ""
   resources:
   - configmaps
+  - persistentvolumeclaims
+  - pods
+  - secrets
+  - services
   verbs:
   - create
   - delete
@@ -25,29 +28,14 @@ rules:
   - create
   - patch
   - update
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - ""
   resources:
-  - secrets
+  - persistentvolumes
   verbs:
-  - create
-  - delete
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - ""
@@ -95,9 +83,18 @@ rules:
   - get
   - update
 - apiGroups:
-  - ""
+  - kueue.x-k8s.io
   resources:
-  - persistentvolumeclaims
+  - resourceflavors
+  - workloadpriorityclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - kueue.x-k8s.io
+  resources:
+  - workloads
   verbs:
   - create
   - delete
@@ -107,25 +104,19 @@ rules:
   - update
   - watch
 - apiGroups:
-  - ""
+  - kueue.x-k8s.io
   resources:
-  - persistentvolumes
+  - workloads/finalizers
   verbs:
-  - get
-  - list
-  - watch
+  - update
 - apiGroups:
-  - ""
+  - kueue.x-k8s.io
   resources:
-  - services
+  - workloads/status
   verbs:
-  - create
-  - delete
   - get
-  - list
   - patch
   - update
-  - watch
 - apiGroups:
   - monitoring.coreos.com
   resources:
@@ -158,21 +149,18 @@ rules:
   - update
   - watch
 - apiGroups:
-  - serving.kserve.io
+  - scheduling.k8s.io
   resources:
-  - inferenceservices
+  - priorityclasses
   verbs:
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - serving.kserve.io
   resources:
-  - inferenceservices/finalizers
+  - inferenceservices
   verbs:
-  - delete
   - get
   - list
   - patch
@@ -181,9 +169,8 @@ rules:
 - apiGroups:
   - serving.kserve.io
   resources:
-  - servingruntimes
+  - inferenceservices/finalizers
   verbs:
-  - create
   - delete
   - get
   - list
@@ -193,15 +180,7 @@ rules:
 - apiGroups:
   - serving.kserve.io
   resources:
-  - servingruntimes/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - trustyai.opendatahub.io
-  resources:
-  - lmevaljobs
+  - servingruntimes
   verbs:
   - create
   - delete
@@ -211,22 +190,17 @@ rules:
   - update
   - watch
 - apiGroups:
-  - trustyai.opendatahub.io
-  resources:
-  - lmevaljobs/finalizers
-  verbs:
-  - update
-- apiGroups:
-  - trustyai.opendatahub.io
+  - serving.kserve.io
   resources:
-  - lmevaljobs/status
+  - servingruntimes/status
   verbs:
   - get
   - patch
   - update
 - apiGroups:
   - trustyai.opendatahub.io
   resources:
+  - lmevaljobs
   - trustyaiservices
   verbs:
   - create
@@ -239,12 +213,14 @@ rules:
 - apiGroups:
   - trustyai.opendatahub.io
   resources:
+  - lmevaljobs/finalizers
   - trustyaiservices/finalizers
   verbs:
   - update
 - apiGroups:
   - trustyai.opendatahub.io
   resources:
+  - lmevaljobs/status
   - trustyaiservices/status
   verbs:
   - get