Skip to content

Commit 3352dfe

Browse files
ruivieiraruivieira
committed
Merge remote-tracking branch 'upstream/main' into RHOAIENG-18409-s3-support
# Conflicts: # cmd/lmes_driver/main.go # controllers/lmes/driver/driver.go
2 parents f4790f1 + 8fb6117 commit 3352dfe

File tree

13 files changed

+200
-2073
lines changed

13 files changed

+200
-2073
lines changed

‎.github/workflows/smoke.yaml

Lines changed: 55 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,62 @@
11
name: Smoke test
22

33
on:
4-
pull_request:
5-
branches:
6-
- main
4+
pull_request:
5+
branches:
6+
- main
77

88
jobs:
9-
deploy:
10-
runs-on: ubuntu-latest
11-
steps:
12-
- name: Checkout
13-
uses: actions/checkout@v2
14-
15-
- name: Set up Docker Buildx
16-
uses: docker/setup-buildx-action@v1
17-
18-
- name: Build and push operator container image
19-
uses: docker/build-push-action@v2
20-
with:
21-
context: .
22-
file: ./Dockerfile
23-
load: true
24-
tags: smoke/operator:pr-${{ github.event.pull_request.number || env.PR_NUMBER }}
25-
26-
- name: Create k8s Kind Cluster
27-
uses: helm/kind-action@v1
28-
with:
29-
node_image: kindest/node:v1.24.17
30-
cluster_name: kind
31-
32-
- name: Load the operator image into Kind
33-
run: |
34-
kind load docker-image smoke/operator:pr-${{ github.event.pull_request.number || env.PR_NUMBER }}
35-
36-
- name: Install kustomize
37-
run: |
38-
curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
39-
sudo mv kustomize /usr/local/bin/
40-
41-
- name: Apply CRDs
42-
run: |
43-
kubectl apply -f tests/crds/monitoring.coreos.com_servicemonitors.yaml
44-
kubectl apply -f tests/crds/route_crd.yaml
45-
kubectl apply -f tests/crds/serving.kserve.io_inferenceservices.yaml
46-
kubectl apply -f tests/crds/serving.kserve.io_servingruntimes.yaml
47-
kustomize build config/crd | kubectl apply -f -
48-
49-
- name: Update params.env file
50-
run: |
51-
sed -i 's|trustyaiOperatorImage=quay.io/trustyai/trustyai-service-operator:latest|trustyaiOperatorImage=smoke/operator:pr-${{ github.event.pull_request.number || env.PR_NUMBER }}|' config/base/params.env
52-
53-
- name: Deploy the operator with kustomize
54-
run: |
55-
kubectl create namespace system
56-
kustomize build config/base | kubectl apply -n system -f -
57-
58-
- name: Run smoke tests
59-
run: ./tests/smoke/test_smoke.sh
60-
9+
deploy:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout
13+
uses: actions/checkout@v2
14+
15+
- name: Set up Docker Buildx
16+
uses: docker/setup-buildx-action@v1
17+
18+
- name: Build and push operator container image
19+
uses: docker/build-push-action@v2
20+
with:
21+
context: .
22+
file: ./Dockerfile
23+
load: true
24+
tags: smoke/operator:pr-${{ github.event.pull_request.number || env.PR_NUMBER }}
25+
26+
- name: Create k8s Kind Cluster
27+
uses: helm/kind-action@v1
28+
with:
29+
node_image: kindest/node:v1.24.17
30+
cluster_name: kind
31+
32+
- name: Load the operator image into Kind
33+
run: |
34+
kind load docker-image smoke/operator:pr-${{ github.event.pull_request.number || env.PR_NUMBER }}
35+
36+
- name: Install kustomize
37+
run: |
38+
curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
39+
sudo mv kustomize /usr/local/bin/
40+
41+
- name: Apply CRDs
42+
run: |
43+
kubectl apply -f tests/crds/monitoring.coreos.com_servicemonitors.yaml
44+
kubectl apply -f tests/crds/route_crd.yaml
45+
kubectl apply -f tests/crds/serving.kserve.io_inferenceservices.yaml
46+
kustomize build config/crd | kubectl apply -f -
47+
48+
- name: Update params.env file
49+
run: |
50+
sed -i 's|trustyaiOperatorImage=quay.io/trustyai/trustyai-service-operator:latest|trustyaiOperatorImage=smoke/operator:pr-${{ github.event.pull_request.number || env.PR_NUMBER }}|' config/base/params.env
51+
52+
- name: Deploy the operator with kustomize
53+
run: |
54+
kubectl create namespace system
55+
kustomize build config/base | kubectl apply -n system -f -
56+
57+
- name: Run smoke tests
58+
run: ./tests/smoke/test_smoke.sh
6159

6260
env:
63-
PR_NUMBER: "default-pr-number"
64-
KUBECONFIG: "${HOME}/.kube/config"
61+
PR_NUMBER: "default-pr-number"
62+
KUBECONFIG: "${HOME}/.kube/config"

‎cmd/lmes_driver/main.go

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,16 @@ func (t *strArrayArg) String() string {
5050
}
5151

5252
var (
53-
taskRecipes strArrayArg
54-
customCards strArrayArg
55-
copy = flag.String("copy", "", "copy this binary to specified destination path")
56-
getStatus = flag.Bool("get-status", false, "Get current status")
57-
shutdown = flag.Bool("shutdown", false, "Shutdown the driver")
58-
outputPath = flag.String("output-path", OutputPath, "output path")
59-
detectDevice = flag.Bool("detect-device", false, "detect available device(s), CUDA or CPU")
53+
taskRecipes strArrayArg
54+
customCards strArrayArg
55+
copy = flag.String("copy", "", "copy this binary to specified destination path")
56+
getStatus = flag.Bool("get-status", false, "Get current status")
57+
shutdown = flag.Bool("shutdown", false, "Shutdown the driver")
58+
outputPath = flag.String("output-path", OutputPath, "output path")
59+
detectDevice = flag.Bool("detect-device", false, "detect available device(s), CUDA or CPU")
60+
commPort = flag.Int("listen-port", driver.DefaultPort, "driver serves APIs on the port")
6061
downloadAssetsS3 = flag.Bool("download-assets-s3", false, "Download assets from S3")
61-
driverLog = ctrl.Log.WithName("driver")
62+
driverLog = ctrl.Log.WithName("driver")
6263
)
6364

6465
func init() {
@@ -106,13 +107,14 @@ func main() {
106107
}
107108

108109
driverOpt := driver.DriverOption{
109-
Context: ctx,
110-
OutputPath: *outputPath,
111-
DetectDevice: *detectDevice,
112-
Logger: driverLog,
113-
TaskRecipes: taskRecipes,
114-
CustomCards: customCards,
115-
Args: args,
110+
Context: ctx,
111+
OutputPath: *outputPath,
112+
DetectDevice: *detectDevice,
113+
Logger: driverLog,
114+
TaskRecipes: taskRecipes,
115+
CustomCards: customCards,
116+
Args: args,
117+
CommPort: *commPort,
116118
DownloadAssetsS3: *downloadAssetsS3,
117119
}
118120

‎config/rbac/role.yaml

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -208,26 +208,6 @@ rules:
208208
- patch
209209
- update
210210
- watch
211-
- apiGroups:
212-
- serving.kserve.io
213-
resources:
214-
- servingruntimes
215-
verbs:
216-
- create
217-
- delete
218-
- get
219-
- list
220-
- patch
221-
- update
222-
- watch
223-
- apiGroups:
224-
- serving.kserve.io
225-
resources:
226-
- servingruntimes/status
227-
verbs:
228-
- get
229-
- patch
230-
- update
231211
- apiGroups:
232212
- trustyai.opendatahub.io
233213
resources:

‎controllers/lmes/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"time"
2525

2626
"github.com/go-logr/logr"
27+
"github.com/trustyai-explainability/trustyai-service-operator/controllers/lmes/driver"
2728
corev1 "k8s.io/api/core/v1"
2829
)
2930

@@ -39,6 +40,7 @@ var Options *serviceOptions = &serviceOptions{
3940
DefaultBatchSize: DefaultBatchSize,
4041
AllowOnline: false,
4142
AllowCodeExecution: false,
43+
DriverPort: driver.DefaultPort,
4244
}
4345

4446
type serviceOptions struct {
@@ -51,6 +53,7 @@ type serviceOptions struct {
5153
DetectDevice bool
5254
AllowOnline bool
5355
AllowCodeExecution bool
56+
DriverPort int
5457
}
5558

5659
func constructOptionsFromConfigMap(log *logr.Logger, configmap *corev1.ConfigMap) error {

‎controllers/lmes/constants.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ const (
3636
DetectDeviceKey = "lmes-detect-device"
3737
AllowOnline = "lmes-allow-online"
3838
AllowCodeExecution = "lmes-allow-code-execution"
39+
DriverPort = "lmes-driver-port"
3940
DefaultPodImage = "quay.io/trustyai/ta-lmes-job:latest"
4041
DefaultDriverImage = "quay.io/trustyai/ta-lmes-driver:latest"
4142
DefaultPodCheckingInterval = time.Second * 10

‎controllers/lmes/driver/driver.go

Lines changed: 21 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ var (
4242
)
4343

4444
const (
45-
// put the domain socket under /tmp. may move to emptydir to share across containers
46-
socketPath = "/tmp/ta-lmes-driver.sock"
45+
// the default port for the driver to listen on
46+
DefaultPort = 18080
4747
DefaultTaskRecipesPath = "/opt/app-root/src/my_tasks"
4848
DefaultCatalogPath = "/opt/app-root/src/my_catalogs"
4949
TaskRecipePrefix = "tr"
@@ -53,16 +53,16 @@ const (
5353
)
5454

5555
type DriverOption struct {
56-
Context context.Context
57-
OutputPath string
58-
DetectDevice bool
59-
TaskRecipesPath string
60-
TaskRecipes []string
61-
CatalogPath string
62-
CustomCards []string
63-
Logger logr.Logger
64-
Args []string
65-
SocketPath string
56+
Context context.Context
57+
OutputPath string
58+
DetectDevice bool
59+
TaskRecipesPath string
60+
TaskRecipes []string
61+
CatalogPath string
62+
CustomCards []string
63+
Logger logr.Logger
64+
Args []string
65+
CommPort int
6666
DownloadAssetsS3 bool
6767
}
6868

@@ -77,7 +77,7 @@ type Driver interface {
7777
type driverComm struct {
7878
connection chan int
7979
server *http.Server
80-
path string
80+
port int
8181
}
8282

8383
type driverImpl struct {
@@ -105,8 +105,8 @@ func NewDriver(opt *DriverOption) (Driver, error) {
105105
opt.CatalogPath = DefaultCatalogPath
106106
}
107107

108-
if opt.SocketPath == "" {
109-
opt.SocketPath = socketPath
108+
if opt.CommPort == 0 {
109+
opt.CommPort = DefaultPort
110110
}
111111

112112
return &driverImpl{
@@ -144,8 +144,8 @@ func (d *driverImpl) Run() error {
144144
}
145145

146146
func (d *driverImpl) GetStatus() (*lmesv1alpha1.LMEvalJobStatus, error) {
147-
client := createClient(d.Option.SocketPath)
148-
resp, err := client.Get(fmt.Sprintf("http://unix%s", GetStatusURI))
147+
client := &http.Client{}
148+
resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/%s", d.Option.CommPort, GetStatusURI))
149149
if err != nil {
150150
return nil, err
151151
}
@@ -161,8 +161,8 @@ func (d *driverImpl) GetStatus() (*lmesv1alpha1.LMEvalJobStatus, error) {
161161
return &status, err
162162
}
163163
func (d *driverImpl) Shutdown() error {
164-
client := createClient(d.Option.SocketPath)
165-
resp, err := client.Post(fmt.Sprintf("http://unix%s", ShutdownURI), "application/json", nil)
164+
client := &http.Client{}
165+
resp, err := client.Post(fmt.Sprintf("http://127.0.0.1:%d/%s", d.Option.CommPort, ShutdownURI), "application/json", nil)
166166
if err != nil {
167167
return err
168168
}
@@ -241,7 +241,7 @@ func (d *driverImpl) setupComm() error {
241241
d.comm = &driverComm{
242242
server: &http.Server{Handler: serve},
243243
connection: make(chan int),
244-
path: d.Option.SocketPath,
244+
port: d.Option.CommPort,
245245
}
246246

247247
// handle the `GetStatus` API: return the complete lmesv1alpha1.LMEvalJobStatus
@@ -268,22 +268,12 @@ func (d *driverImpl) setupComm() error {
268268
return nil
269269
}
270270

271-
func createClient(path string) *http.Client {
272-
return &http.Client{
273-
Transport: &http.Transport{
274-
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
275-
return net.Dial("unix", path)
276-
},
277-
},
278-
}
279-
}
280-
281271
func (dc *driverComm) wait4Sutdownload() {
282272
<-dc.connection
283273
}
284274

285275
func (dc *driverComm) serve() error {
286-
socket, err := net.Listen("unix", dc.path)
276+
socket, err := net.Listen("tcp4", fmt.Sprintf("127.0.0.1:%d", dc.port))
287277
if err != nil {
288278
return err
289279
}
@@ -295,7 +285,6 @@ func (dc *driverComm) close() {
295285
if dc.server != nil && dc.connection != nil {
296286
dc.server.Shutdown(context.Background())
297287
close(dc.connection)
298-
os.Remove(dc.path)
299288
}
300289
}
301290

0 commit comments

Comments
 (0)