@@ -21,8 +21,11 @@ import (
21
21
"context"
22
22
"errors"
23
23
"fmt"
24
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24
25
"path"
26
+ "strconv"
25
27
"strings"
28
+ "time"
26
29
27
30
"github.com/chaosblade-io/chaosblade-spec-go/spec"
28
31
"github.com/sirupsen/logrus"
@@ -153,9 +156,102 @@ func (e *ExecCommandInPodExecutor) execInMatchedPod(ctx context.Context, expMode
153
156
}
154
157
experimentStatus .Success = success
155
158
experimentStatus .ResStatuses = statuses
159
+
160
+ checkExperimentStatus (ctx , expModel , statuses , experimentIdentifiers , e )
156
161
return spec .ReturnResultIgnoreCode (experimentStatus )
157
162
}
158
163
164
+ func checkExperimentStatus (ctx context.Context , expModel * spec.ExpModel , statuses []v1alpha1.ResourceStatus , identifiers []ExperimentIdentifierInPod , e * ExecCommandInPodExecutor ) {
165
+ tt := expModel .ActionFlags ["timeout" ]
166
+ if _ , ok := spec .IsDestroy (ctx ); ! ok && tt != "" && len (statuses ) > 0 {
167
+ experimentId := GetExperimentIdFromContext (ctx )
168
+ go func () {
169
+ timeout , err := strconv .ParseUint (tt , 10 , 64 )
170
+ if err != nil {
171
+ // the err checked in RunE function
172
+ timeDuartion , _ := time .ParseDuration (tt )
173
+ timeout = uint64 (timeDuartion .Seconds ())
174
+ }
175
+ time .Sleep (time .Duration (timeout ) * time .Second )
176
+
177
+ ctx , cancel := context .WithTimeout (context .Background (), time .Second * 30 )
178
+ defer cancel ()
179
+
180
+ ticker := time .NewTicker (time .Second )
181
+ TickerLoop:
182
+ for range ticker .C {
183
+ select {
184
+ case <- ctx .Done ():
185
+ ticker .Stop ()
186
+ break TickerLoop
187
+ default :
188
+ isDestroyed := true
189
+ for i , status := range statuses {
190
+ if ! status .Success {
191
+ continue
192
+ }
193
+ containerObjectMeta := ParseIdentifier (status .Identifier )
194
+ identifier := identifiers [i ]
195
+ podName := containerObjectMeta .PodName
196
+ podNamespace := containerObjectMeta .Namespace
197
+ containerName := containerObjectMeta .ContainerName
198
+ if identifier .ChaosBladePodName != "" {
199
+ podName = identifier .ChaosBladePodName
200
+ podNamespace = identifier .ChaosBladeNamespace
201
+ containerName = identifier .ChaosBladeContainerName
202
+ }
203
+ response := e .Client .Exec (& channel.ExecOptions {
204
+ StreamOptions : channel.StreamOptions {
205
+ ErrDecoder : func (bytes []byte ) interface {} {
206
+ content := string (bytes )
207
+ return spec .Decode (content , spec .ReturnFail (spec .Code [spec .K8sInvokeError ], content ))
208
+ },
209
+ OutDecoder : func (bytes []byte ) interface {} {
210
+ content := string (bytes )
211
+ return spec .Decode (content , spec .ReturnFail (spec .Code [spec .K8sInvokeError ], content ))
212
+ },
213
+ },
214
+ PodName : podName ,
215
+ PodNamespace : podNamespace ,
216
+ ContainerName : containerName ,
217
+ Command : []string {getTargetChaosBladeBin (expModel ), "status" , status .Id },
218
+ IgnoreOutput : false ,
219
+ }).(* spec.Response )
220
+ if response .Success {
221
+ result := response .Result .(map [string ]interface {})
222
+ if result ["Status" ] != v1alpha1 .DestroyedState {
223
+ isDestroyed = false
224
+ break
225
+ }
226
+ } else {
227
+ isDestroyed = false
228
+ break
229
+ }
230
+ }
231
+
232
+ if isDestroyed {
233
+ logrus .Info ("The experiment was destroyed, ExperimentId: " , experimentId )
234
+ cli := e .Client .Client
235
+ objectMeta := metav1.ObjectMeta {Name : experimentId }
236
+ err := cli .Delete (context .TODO (), & v1alpha1.ChaosBlade {
237
+ TypeMeta : metav1.TypeMeta {
238
+ APIVersion : "chaosblade.io/v1alpha1" ,
239
+ Kind : "ChaosBlade" ,
240
+ },
241
+ ObjectMeta : objectMeta ,
242
+ })
243
+ if err != nil {
244
+ logrus .Warn (err .Error ())
245
+ } else {
246
+ ticker .Stop ()
247
+ }
248
+ }
249
+ }
250
+ }
251
+ }()
252
+ }
253
+ }
254
+
159
255
func (e * ExecCommandInPodExecutor ) execCommands (ctx context.Context , rsStatus v1alpha1.ResourceStatus ,
160
256
identifier ExperimentIdentifierInPod , statuses []v1alpha1.ResourceStatus ) (bool , []v1alpha1.ResourceStatus ) {
161
257
success := false
0 commit comments