Fix some integration test issues when using rancher-desktop

Signed-off-by: Thomas Hallgren <[email protected]>

Author: thallgren

integration_test/itest/cluster.go

@@ -195,6 +195,21 @@ func (s *cluster) Initialize(ctx context.Context) context.Context {
 		s.agentVersion = s.managerVersion
 	}
 
+	// We cannot use t.TempDir() here, because it will not mount correctly in
+	// rancher-desktop and docker-desktop (unless they are configured to allow
+	// mounts directly from /tmp). So we use a tempdir in BUILD_OUTPUT instead
+	// because it's believed to be both mountable and writable.
+	tempDir := dos.Getenv(ctx, "TELEPRESENCE_TEMP_DIR")
+	if tempDir == "" {
+		tempDir = filepath.Join(BuildOutput(ctx), "tmp")
+	}
+	_ = dos.RemoveAll(ctx, tempDir)
+	require.NoError(t, dos.MkdirAll(ctx, tempDir, 0o777))
+	ctx = withTempDirBase(ctx, &tempDirBase{tempDir: tempDir})
+	t.Cleanup(func() {
+		_ = os.RemoveAll(tempDir)
+	})
+
 	registry := dos.Getenv(ctx, "TELEPRESENCE_REGISTRY")
 	if registry == "" {
 		registry = "ghcr.io/telepresenceio"
@@ -290,11 +305,8 @@ func (s *cluster) Initialize(ctx context.Context) context.Context {
 
 	s.ensureQuit(ctx)
 	s.ensureNoManager(ctx)
-	_ = Run(ctx, "kubectl", "delete", "ns", "-l", AssignPurposeLabel)
 	_ = Run(ctx, "kubectl", "delete", "-f", filepath.Join("testdata", "k8s", "client_rbac.yaml"))
-	_ = Run(ctx, "kubectl", "delete", "ns", "-l", AssignPurposeLabel)
-	_ = Run(ctx, "kubectl", "delete", "pv", "-l", AssignPurposeLabel)
-	_ = Run(ctx, "kubectl", "delete", "storageclass", "-l", AssignPurposeLabel)
+	_ = Run(ctx, "kubectl", "delete", "all", "-l", AssignPurposeLabel)
 	return ctx
 }
 
@@ -354,10 +366,7 @@ func (s *cluster) tearDown(ctx context.Context) {
 	if s.kubeConfig != "" {
 		ctx = WithWorkingDir(ctx, GetOSSRoot(ctx))
 		_ = Run(ctx, "kubectl", "delete", "-f", filepath.Join("testdata", "k8s", "client_rbac.yaml"))
-		_ = Run(ctx, "kubectl", "delete", "--wait=false", "ns", "-l", AssignPurposeLabel)
-		_ = Run(ctx, "kubectl", "delete", "--wait=false", "pv", "-l", AssignPurposeLabel)
-		_ = Run(ctx, "kubectl", "delete", "--wait=false", "storageclass", "-l", AssignPurposeLabel)
-		_ = Run(ctx, "kubectl", "delete", "--wait=false", "mutatingwebhookconfigurations", "-l", AssignPurposeLabel)
+		_ = Run(ctx, "kubectl", "delete", "--wait=false", "all", "-l", AssignPurposeLabel)
 	}
 }
 
@@ -439,7 +448,7 @@ func (s *cluster) withBasicConfig(c context.Context, t *testing.T) context.Conte
 	require.NoError(t, err)
 	configYamlStr := string(configYaml)
 
-	configDir := t.TempDir()
+	configDir := TempDir(c)
 	c = filelocation.WithAppUserConfigDir(c, configDir)
 	c, err = SetConfig(c, configDir, configYamlStr)
 	require.NoError(t, err)
@@ -1071,7 +1080,7 @@ func WithConfig(c context.Context, modifierFunc func(config client.Config)) cont
 	configYaml, err := configCopy.(client.Config).MarshalYAML()
 	require.NoError(t, err)
 	configYamlStr := string(configYaml)
-	configDir, err := os.MkdirTemp(t.TempDir(), "config")
+	configDir, err := os.MkdirTemp(TempDir(c), "config")
 	require.NoError(t, err)
 	c, err = SetConfig(c, configDir, configYamlStr)
 	require.NoError(t, err)

integration_test/itest/namespace.go

@@ -103,7 +103,14 @@ func (s *nsPair) setup(ctx context.Context) bool {
 		return false
 	}
 	err := Kubectl(ctx, s.Namespace, "apply", "-f", filepath.Join(GetOSSRoot(ctx), "testdata", "k8s", "client_sa.yaml"))
-	assert.NoError(t, err, "failed to create connect ServiceAccount")
+	if assert.NoError(t, err, "failed to create connect ServiceAccount") {
+		db, err := ReadTemplate(ctx, filepath.Join("testdata", "k8s", "client_rancher.goyaml"), map[string]string{
+			"ManagerNamespace": s.Namespace,
+		})
+		if assert.NoError(t, err) {
+			assert.NoError(t, Kubectl(dos.WithStdin(ctx, bytes.NewReader(db)), s.Namespace, "apply", "-f", "-"))
+		}
+	}
 	return !t.Failed()
 }
 

integration_test/itest/tempdir.go

@@ -0,0 +1,37 @@
+package itest
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync/atomic"
+
+	"github.com/stretchr/testify/require"
+)
+
+type tempDirBase struct {
+	tempDir    string
+	tempDirSeq uint64
+}
+
+type tempDirBaseKey struct{}
+
+func withTempDirBase(ctx context.Context, td *tempDirBase) context.Context {
+	return context.WithValue(ctx, tempDirBaseKey{}, td)
+}
+
+// TempDir returns a temporary directory for the test to use.
+// The directory is automatically removed when the test and
+// all its subtests complete.
+// Each subsequent call to t.TempDir returns a unique directory;
+// if the directory creation fails, TempDir terminates the test by calling Fatal.
+func TempDir(ctx context.Context) string {
+	t := getT(ctx)
+	if td, ok := ctx.Value(tempDirBaseKey{}).(*tempDirBase); ok {
+		seq := atomic.AddUint64(&td.tempDirSeq, 1)
+		dir := fmt.Sprintf("%s%c%03d", td.tempDir, os.PathSeparator, seq)
+		require.NoError(t, os.Mkdir(dir, 0o777))
+		return dir
+	}
+	return t.TempDir()
+}

integration_test/testdata/k8s/client_rancher.goyaml

@@ -0,0 +1,27 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name:  rancher-inspect
+  labels:
+    purpose: tp-cli-testing
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rancher-inspect
+  labels:
+    purpose: tp-cli-testing
+subjects:
+  - kind: ServiceAccount
+    name: telepresence-test-developer
+    namespace: {{ .ManagerNamespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  name: rancher-inspect
+  kind: ClusterRole