@@ -390,6 +390,7 @@ func (config *TestConfig) RemoveNode(nodeName string) error {
390390 if _ , err := RunCommand (cmd ); err != nil {
391391 return fmt .Errorf ("failed to remove node %s: %v" , nodeName , err )
392392 }
393+ fmt .Println ("Stopped and removed " , nodeName )
393394 return nil
394395}
395396
@@ -428,13 +429,15 @@ func (config *TestConfig) Cleanup() error {
428429 errs = append (errs , err )
429430 }
430431 }
432+ config .Servers = nil
431433
432434 // Stop and remove all agents
433435 for _ , agent := range config .Agents {
434436 if err := config .RemoveNode (agent .Name ); err != nil {
435437 errs = append (errs , err )
436438 }
437439 }
440+ config .Agents = nil
438441
439442 // Stop DB if it was started
440443 if config .DBType == "mysql" || config .DBType == "postgres" {
@@ -456,8 +459,6 @@ func (config *TestConfig) Cleanup() error {
456459 if config .TestDir != "" {
457460 return os .RemoveAll (config .TestDir )
458461 }
459- config .Agents = nil
460- config .Servers = nil
461462 return nil
462463}
463464
@@ -640,10 +641,47 @@ func FetchExternalIPs(kubeconfig string, servicename string) ([]string, error) {
640641// RestartCluster restarts the k3s service on each node given
641642func RestartCluster (nodes []DockerNode ) error {
642643 for _ , node := range nodes {
643- cmd := "systemctl restart k3s* --all"
644+ // Wait 60 seconds for the restart to succeed.
645+ // If k3s doesn't report started to systemd within 60 seconds something is wrong.
646+ cmd := "timeout -v 60 systemctl restart k3s* --all"
644647 if _ , err := node .RunCmdOnNode (cmd ); err != nil {
645648 return err
646649 }
647650 }
648651 return nil
649652}
653+
654+ func DescribeNodesAndPods (config * TestConfig ) string {
655+ cmd := "kubectl describe node,pod -A --kubeconfig=" + config .KubeconfigFile
656+ out , err := RunCommand (cmd )
657+ if err != nil {
658+ return fmt .Sprintf ("** %v **\n %s" , err , out )
659+ }
660+ return out
661+ }
662+
663+ func TailDockerLogs (lines int , nodes []DockerNode ) string {
664+ logs := & strings.Builder {}
665+ for _ , node := range nodes {
666+ cmd := fmt .Sprintf ("docker logs %s --tail=%d" , node .Name , lines )
667+ if l , err := RunCommand (cmd ); err != nil {
668+ fmt .Fprintf (logs , "** failed to read docker logs for node %s ***\n %v\n " , node .Name , err )
669+ } else {
670+ fmt .Fprintf (logs , "** docker logs for node %s ***\n %s\n " , node .Name , l )
671+ }
672+ }
673+ return logs .String ()
674+ }
675+
676+ func TailJournalLogs (lines int , nodes []DockerNode ) string {
677+ logs := & strings.Builder {}
678+ for _ , node := range nodes {
679+ cmd := fmt .Sprintf ("journalctl -u k3s* --no-pager --lines=%d" , lines )
680+ if l , err := node .RunCmdOnNode (cmd ); err != nil {
681+ fmt .Fprintf (logs , "** failed to read journald log for node %s ***\n %v\n " , node .Name , err )
682+ } else {
683+ fmt .Fprintf (logs , "** journald log for node %s ***\n %s\n " , node .Name , l )
684+ }
685+ }
686+ return logs .String ()
687+ }
0 commit comments