Implement automatic reconnection for SSH session (#22178)

Implement automatic reconnection for SSH session

Author: KevinFairise2

test/new-e2e/pkg/components/remotehost.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
@@ -27,7 +28,7 @@ import (
 const (
 	// Waiting for only 10s as we expect remote to be ready when provisioning
 	sshRetryInterval = 2 * time.Second
-	sshMaxRetries    = 5
+	sshMaxRetries    = 20
 )
 
 // RemoteHost represents a remote host
@@ -43,21 +44,33 @@ var _ e2e.Initializable = &RemoteHost{}
 // Init is called by e2e test Suite after the component is provisioned.
 func (h *RemoteHost) Init(ctx e2e.Context) error {
 	h.context = ctx
-	return h.ReconnectSSH()
+	return h.reconnectSSH()
 }
 
 // Execute executes a command and returns an error if any.
 func (h *RemoteHost) Execute(command string, options ...ExecuteOption) (string, error) {
+	var err error
+	var output string
+
 	params, err := optional.MakeParams(options...)
 	if err != nil {
 		return "", err
 	}
+
 	cmd := h.buildEnvVariables(command, params.EnvVariables)
+	output, err = clients.ExecuteCommand(h.client, cmd)
 
-	output, err := clients.ExecuteCommand(h.client, cmd)
+	if err != nil && strings.Contains(err.Error(), "failed to create session:") {
+		err = h.reconnectSSH()
+		if err != nil {
+			return "", err
+		}
+		output, err = clients.ExecuteCommand(h.client, cmd)
+	}
 	if err != nil {
 		return "", fmt.Errorf("%v: %v", output, err)
 	}
+
 	return output, nil
 }
 
@@ -130,9 +143,9 @@ func (h *RemoteHost) RemoveAll(path string) error {
 	return clients.RemoveAll(h.client, path)
 }
 
-// ReconnectSSH recreate the SSH connection to the VM. Should be used only after VM reboot to restore the SSH connection.
+// reconnectSSH recreate the SSH connection to the VM. Should be used only after VM reboot to restore the SSH connection.
 // Returns an error if the VM is not reachable after retries.
-func (h *RemoteHost) ReconnectSSH() error {
+func (h *RemoteHost) reconnectSSH() error {
 	h.context.T().Logf("connecting to remote VM at %s@%s", h.Username, h.Address)
 
 	if h.client != nil {

test/new-e2e/pkg/utils/clients/ssh.go

@@ -89,7 +89,7 @@ func getSSHClient(user, host string, privateKey, privateKeyPassphrase []byte) (*
 func ExecuteCommand(client *ssh.Client, command string) (string, error) {
 	session, err := client.NewSession()
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to create session: %v", err)
 	}
 
 	stdout, err := session.CombinedOutput(command)