Add lm-eval-service controller (#258)

* feat: Initial database support (#246)

* Initial database support

- Add status checking
- Add better storage flags
- Add spec.storage.format validation
- Add DDL
-Add HIBERNATE format to DB (test)
- Update service image
- Revert identifier to DATABASE
- Update CR options (remove mandatory data)

* Remove default DDL generation env var

* Update service image to latest tag

* Add migration awareness

* Add updating pods for migration

* Change JDBC url from mysql to mariadb

* Fix TLS mount

* Revert images

* Remove redundant logic

* Fix comments

* feat: Add TLS certificate mount on ModelMesh (#255)

* feat: Add TLS certificate mount on ModelMesh

* Revert from http to https until kserve/modelmesh#147 is merged

* Add lm-eval-service controller

refactor the existing TrustyAIService controller and
add LMEvalService controller

Signed-off-by: Yihong Wang <[email protected]>

---------

Signed-off-by: Yihong Wang <[email protected]>
Co-authored-by: Rui Vieira <[email protected]>

Author: yhwang

.github/workflows/controller-tests.yaml

@@ -13,7 +13,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.19.0'
+          go-version: '1.21.12'
 
       - name: Download & install envtest binaries
         run: |

.yamllint.yaml

@@ -6,4 +6,6 @@ rules:
     level: warning
   hyphens:
     max-spaces-after: 1
-    level: warning
+    level: warning
+  indentation:
+    indent-sequences: consistent

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+FROM registry.access.redhat.com/ubi8/go-toolset:1.21 AS builder
 ARG TARGETOS
 ARG TARGETARCH
 
@@ -12,7 +12,7 @@ COPY go.sum go.sum
 RUN go mod download
 
 # Copy the go source
-COPY main.go main.go
+COPY cmd/ cmd/
 COPY api/ api/
 COPY controllers/ controllers/
 
@@ -22,7 +22,7 @@ COPY controllers/ controllers/
 # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
 # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
 USER root
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/operator/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Dockerfile.driver

@@ -0,0 +1,16 @@
+FROM registry.access.redhat.com/ubi8/go-toolset:1.21 AS builder
+
+WORKDIR /go/src/github.com/trustyai-explainability/trustyai-service-operator
+COPY . .
+
+RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./cmd/lmes_driver/*.go
+
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+
+COPY --from=builder /bin/driver /bin/driver
+
+USER 65532:65532
+
+WORKDIR /bin
+
+ENTRYPOINT [ "/bin/driver" ]

Makefile

@@ -7,6 +7,9 @@ VERSION ?= 1.17.0
 
 BUILD_TOOL ?= podman
 
+# enable TrustyAIService by default for `make run`
+ENABLED_SERVICES ?= TAS
+
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
 # To re-generate a bundle for other specific channels without changing the standard setup, you can:
@@ -111,11 +114,11 @@ test: manifests generate fmt vet envtest ## Run tests.
 
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
-	go build -o bin/manager main.go
+	go build -o bin/manager cmd/operator/main.go
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./main.go
+	go run ./cmd/operator/main.go --enable-services $(ENABLED_SERVICES)
 
 # If you wish built the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.

PROJECT

@@ -4,7 +4,7 @@
 # More info: https://book.kubebuilder.io/reference/project-config.html
 domain: opendatahub.io
 layout:
-- go.kubebuilder.io/v3
+- go.kubebuilder.io/v4
 plugins:
   manifests.sdk.operatorframework.io/v2: {}
   scorecard.sdk.operatorframework.io/v2: {}
@@ -18,6 +18,19 @@ resources:
   domain: opendatahub.io
   group: trustyai
   kind: TrustyAIService
-  path: github.com/trustyai-explainability/trustyai-service-operator/api/v1alpha1
+  path: github.com/trustyai-explainability/trustyai-service-operator/api/tas/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: opendatahub.io
+  group: trustyai
+  kind: LMEvalJob
+  path: github.com/trustyai-explainability/trustyai-service-operator/api/lmes/v1alpha1
+  version: v1alpha1
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
 version: "3"

api/lmes/v1alpha1/groupversion_info.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1alpha1 contains API Schema definitions for the trustyai.opendatahub.io v1alpha1 API group
+// +kubebuilder:object:generate=true
+// +groupName=trustyai.opendatahub.io
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+const (
+	GroupName     = "trustyai.opendatahub.io"
+	Version       = "v1alpha1"
+	KindName      = "LMEvalJob"
+	FinalizerName = "trustyai.opendatahub.io/lmes-finalizer"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: GroupName, Version: Version}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

api/lmes/v1alpha1/lmevaljob_types.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// Represent a job's status
+// +kubebuilder:validation:Enum=New;Scheduled;Running;Complete;Cancelled
+type JobState string
+
+const (
+	// The job is just created
+	NewJobState JobState = "New"
+	// The job is scheduled and waiting for available resources to run it
+	ScheduledJobState JobState = "Scheduled"
+	// The job is running
+	RunningJobState JobState = "Running"
+	// The job is complete
+	CompleteJobState JobState = "Complete"
+	// The job is cancelled
+	CancelledJobState JobState = "Cancelled"
+)
+
+// +kubebuilder:validation:Enum=NoReason;Succeeded;Failed;Cancelled
+type Reason string
+
+const (
+	// Job is still running and no final result yet
+	NoReason Reason = "NoReason"
+	// Job finished successfully
+	SucceedReason Reason = "Succeeded"
+	// Job failed
+	FailedReason Reason = "Failed"
+	// Job is cancelled
+	CancelledReason Reason = "Cancelled"
+)
+
+type Arg struct {
+	Name  string `json:"name"`
+	Value string `json:"value,omitempty"`
+}
+
+type EnvSecret struct {
+	// Environment's name
+	Env string `json:"env"`
+	// The secret is from a secret object
+	// +optional
+	SecretRef *corev1.SecretKeySelector `json:"secretRef,omitempty"`
+	// The secret is from a plain text
+	// +optional
+	Secret *string `json:"secret,omitempty"`
+}
+
+type FileSecret struct {
+	// The secret object
+	SecretRef corev1.SecretVolumeSource `json:"secretRef,omitempty"`
+	// The path to mount the secret
+	MountPath string `json:"mountPath"`
+}
+
+// LMEvalJobSpec defines the desired state of LMEvalJob
+type LMEvalJobSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Model name
+	Model string `json:"model"`
+	// Args for the model
+	// +optional
+	ModelArgs []Arg `json:"modelArgs,omitempty"`
+	// Evaluation tasks
+	Tasks []string `json:"tasks"`
+	// Sets the number of few-shot examples to place in context
+	// +optional
+	NumFewShot *int `json:"numFewShot,omitempty"`
+	// Accepts an integer, or a float between 0.0 and 1.0 . If passed, will limit
+	// the number of documents to evaluate to the first X documents (if an integer)
+	// per task or first X% of documents per task
+	// +optional
+	Limit string `json:"limit,omitempty"`
+	// Map to `--gen_kwargs` parameter for the underlying library.
+	// +optional
+	GenArgs []Arg `json:"genArgs,omitempty"`
+	// If this flag is passed, then the model's outputs, and the text fed into the
+	// model, will be saved at per-document granularity
+	// +optional
+	LogSamples *bool `json:"logSamples,omitempty"`
+	// Assign secrets to the environment variables
+	// +optional
+	EnvSecrets []EnvSecret `json:"envSecrets,omitempty"`
+	// Use secrets as files
+	FileSecrets []FileSecret `json:"fileSecrets,omitempty"`
+}
+
+// LMEvalJobStatus defines the observed state of LMEvalJob
+type LMEvalJobStatus struct {
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// The name of the Pod that runs the evaluation job
+	// +optional
+	PodName string `json:"podName,omitempty"`
+	// State of the job
+	// +optional
+	State JobState `json:"state,omitempty"`
+	// Final result of the job
+	// +optional
+	Reason Reason `json:"reason,omitempty"`
+	// Message about the current/final status
+	// +optional
+	Message string `json:"message,omitempty"`
+	// Information when was the last time the job was successfully scheduled.
+	// +optional
+	LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"`
+	// Information when the job's state changes to Complete.
+	// +optional
+	CompleteTime *metav1.Time `json:"completeTime,omitempty"`
+	// Evaluation results
+	// +optional
+	Results string `json:"results,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// LMEvalJob is the Schema for the lmevaljobs API
+type LMEvalJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   LMEvalJobSpec   `json:"spec,omitempty"`
+	Status LMEvalJobStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// LMEvalJobList contains a list of LMEvalJob
+type LMEvalJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []LMEvalJob `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&LMEvalJob{}, &LMEvalJobList{})
+}