Skip to content

Commit f0d0351

Browse files
tiny-xxcaspar
authored andcommitted
fixbug: use --timeout flag in k8s environment
Signed-off-by: tiny.x <[email protected]>
1 parent 47bb76a commit f0d0351

File tree

2 files changed

+96
-5
lines changed

2 files changed

+96
-5
lines changed

build/Dockerfile

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,6 @@ FROM alpine:3.8 as builder
33
ENV OPERATOR=/usr/local/bin/chaosblade-operator
44
COPY build/_output/bin/chaosblade-operator /usr/local/bin/
55

6-
RUN wget https://github.com/upx/upx/releases/download/v3.96/upx-3.96-amd64_linux.tar.xz \
7-
&& tar -xvf upx-3.96-amd64_linux.tar.xz \
8-
&& upx-3.96-amd64_linux/upx ${OPERATOR} \
9-
&& rm -rf upx*
10-
116
FROM registry.access.redhat.com/ubi7/ubi-minimal:latest
127

138
ENV OPERATOR=/usr/local/bin/chaosblade-operator \

exec/model/executor.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@ import (
2121
"context"
2222
"errors"
2323
"fmt"
24+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2425
"path"
26+
"strconv"
2527
"strings"
28+
"time"
2629

2730
"github.com/chaosblade-io/chaosblade-spec-go/spec"
2831
"github.com/sirupsen/logrus"
@@ -153,9 +156,102 @@ func (e *ExecCommandInPodExecutor) execInMatchedPod(ctx context.Context, expMode
153156
}
154157
experimentStatus.Success = success
155158
experimentStatus.ResStatuses = statuses
159+
160+
checkExperimentStatus(ctx, expModel, statuses, experimentIdentifiers, e)
156161
return spec.ReturnResultIgnoreCode(experimentStatus)
157162
}
158163

164+
func checkExperimentStatus(ctx context.Context, expModel *spec.ExpModel, statuses []v1alpha1.ResourceStatus, identifiers []ExperimentIdentifierInPod, e *ExecCommandInPodExecutor) {
165+
tt := expModel.ActionFlags["timeout"]
166+
if _, ok := spec.IsDestroy(ctx); !ok && tt != "" && len(statuses) > 0 {
167+
experimentId := GetExperimentIdFromContext(ctx)
168+
go func() {
169+
timeout, err := strconv.ParseUint(tt, 10, 64)
170+
if err != nil {
171+
// the err checked in RunE function
172+
timeDuartion, _ := time.ParseDuration(tt)
173+
timeout = uint64(timeDuartion.Seconds())
174+
}
175+
time.Sleep(time.Duration(timeout) * time.Second)
176+
177+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
178+
defer cancel()
179+
180+
ticker := time.NewTicker(time.Second)
181+
TickerLoop:
182+
for range ticker.C {
183+
select {
184+
case <-ctx.Done():
185+
ticker.Stop()
186+
break TickerLoop
187+
default:
188+
isDestroyed := true
189+
for i, status := range statuses {
190+
if !status.Success {
191+
continue
192+
}
193+
containerObjectMeta := ParseIdentifier(status.Identifier)
194+
identifier := identifiers[i]
195+
podName := containerObjectMeta.PodName
196+
podNamespace := containerObjectMeta.Namespace
197+
containerName := containerObjectMeta.ContainerName
198+
if identifier.ChaosBladePodName != "" {
199+
podName = identifier.ChaosBladePodName
200+
podNamespace = identifier.ChaosBladeNamespace
201+
containerName = identifier.ChaosBladeContainerName
202+
}
203+
response := e.Client.Exec(&channel.ExecOptions{
204+
StreamOptions: channel.StreamOptions{
205+
ErrDecoder: func(bytes []byte) interface{} {
206+
content := string(bytes)
207+
return spec.Decode(content, spec.ReturnFail(spec.Code[spec.K8sInvokeError], content))
208+
},
209+
OutDecoder: func(bytes []byte) interface{} {
210+
content := string(bytes)
211+
return spec.Decode(content, spec.ReturnFail(spec.Code[spec.K8sInvokeError], content))
212+
},
213+
},
214+
PodName: podName,
215+
PodNamespace: podNamespace,
216+
ContainerName: containerName,
217+
Command: []string{getTargetChaosBladeBin(expModel), "status", status.Id},
218+
IgnoreOutput: false,
219+
}).(*spec.Response)
220+
if response.Success {
221+
result := response.Result.(map[string]interface{})
222+
if result["Status"] != v1alpha1.DestroyedState {
223+
isDestroyed = false
224+
break
225+
}
226+
} else {
227+
isDestroyed = false
228+
break
229+
}
230+
}
231+
232+
if isDestroyed {
233+
logrus.Info("The experiment was destroyed, ExperimentId: ", experimentId)
234+
cli := e.Client.Client
235+
objectMeta := metav1.ObjectMeta{Name: experimentId}
236+
err := cli.Delete(context.TODO(), &v1alpha1.ChaosBlade{
237+
TypeMeta: metav1.TypeMeta{
238+
APIVersion: "chaosblade.io/v1alpha1",
239+
Kind: "ChaosBlade",
240+
},
241+
ObjectMeta: objectMeta,
242+
})
243+
if err != nil {
244+
logrus.Warn(err.Error())
245+
} else {
246+
ticker.Stop()
247+
}
248+
}
249+
}
250+
}
251+
}()
252+
}
253+
}
254+
159255
func (e *ExecCommandInPodExecutor) execCommands(ctx context.Context, rsStatus v1alpha1.ResourceStatus,
160256
identifier ExperimentIdentifierInPod, statuses []v1alpha1.ResourceStatus) (bool, []v1alpha1.ResourceStatus) {
161257
success := false

0 commit comments

Comments
 (0)