diff --git a/.gitignore b/.gitignore index 91f8b4c7d8..73f58c145b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,8 @@ bin .envrc .env +.test.env *.pem # OS -.DS_STORE \ No newline at end of file +.DS_STORE diff --git a/Makefile b/Makefile index ea5681b752..a0c7a34cb8 100644 --- a/Makefile +++ b/Makefile @@ -54,9 +54,11 @@ endif all: manager +GO_TEST_ARGS ?= -short + # Run tests test: generate fmt vet manifests - go test ./... -coverprofile cover.out + go test $(GO_TEST_ARGS) ./... -coverprofile cover.out test-with-deps: kube-apiserver etcd kubectl # See https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#pkg-constants diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000000..68f4d84cef --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,148 @@ +package e2e + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/actions-runner-controller/actions-runner-controller/testing" +) + +// If you're willing to run this test via VS Code "run test" or "debug test", +// almost certainly you'd want to make the default go test timeout from 30s to longer and enough value. +// Press Cmd + Shift + P, type "Workspace Settings" and open it, and type "go test timeout" and set e.g. 600s there. +// See https://github.com/golang/vscode-go/blob/master/docs/settings.md#gotesttimeout for more information. +// +// This tests ues testing.Logf extensively for debugging purpose. +// But messages logged via Logf shows up only when the test failed by default. +// To always enable logging, do not forget to pass `-test.v` to `go test`. +// If you're using VS Code, open `Workspace Settings` and search for `go test flags`, edit the `settings.json` and put the below: +// "go.testFlags": ["-v"] +func TestE2E(t *testing.T) { + if testing.Short() { + t.Skip("Skipped as -short is set") + } + + Img := func(repo, tag string) testing.ContainerImage { + return testing.ContainerImage{ + Repo: repo, + Tag: tag, + } + } + + controllerImageRepo := "actionsrunnercontrollere2e/actions-runner-controller" + controllerImageTag := "e2e" + controllerImage := Img(controllerImageRepo, controllerImageTag) + runnerImageRepo := "actionsrunnercontrollere2e/actions-runner" + runnerImageTag := "e2e" + runnerImage := Img(runnerImageRepo, runnerImageTag) + + prebuildImages := []testing.ContainerImage{ + controllerImage, + runnerImage, + } + + builds := []testing.DockerBuild{ + { + Dockerfile: "../../Dockerfile", + Args: []testing.BuildArg{}, + Image: controllerImage, + }, + { + Dockerfile: "../../runner/Dockerfile", + Args: []testing.BuildArg{}, + Image: runnerImage, + }, + } + + certManagerVersion := "v1.1.1" + + images := []testing.ContainerImage{ + Img("docker", "dind"), + Img("quay.io/brancz/kube-rbac-proxy", "v0.10.0"), + Img("quay.io/jetstack/cert-manager-controller", certManagerVersion), + Img("quay.io/jetstack/cert-manager-cainjector", certManagerVersion), + Img("quay.io/jetstack/cert-manager-webhook", certManagerVersion), + } + + k := testing.Start(t, testing.Cluster{}, testing.Preload(images...)) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + t.Run("build images", func(t *testing.T) { + if err := k.BuildImages(ctx, builds); err != nil { + t.Fatal(err) + } + }) + + t.Run("load images", func(t *testing.T) { + if err := k.LoadImages(ctx, prebuildImages); err != nil { + t.Fatal(err) + } + }) + + kubectlEnv := []string{ + "KUBECONFIG=" + k.Kubeconfig(), + } + + t.Run("install cert-manager", func(t *testing.T) { + certmanagerVersion := "v1.1.1" + + if err := k.Apply(ctx, fmt.Sprintf("https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml", certmanagerVersion), testing.KubectlConfig{NoValidate: true}); err != nil { + t.Fatal(err) + } + + certmanagerKubectlCfg := testing.KubectlConfig{ + Env: kubectlEnv, + Namespace: "cert-manager", + Timeout: 90 * time.Second, + } + + if err := k.WaitUntilDeployAvailable(ctx, "cert-manager-cainjector", certmanagerKubectlCfg); err != nil { + t.Fatal(err) + } + + if err := k.WaitUntilDeployAvailable(ctx, "cert-manager-webhook", certmanagerKubectlCfg.WithTimeout(60*time.Second)); err != nil { + t.Fatal(err) + } + + if err := k.WaitUntilDeployAvailable(ctx, "cert-manager", certmanagerKubectlCfg.WithTimeout(60*time.Second)); err != nil { + t.Fatal(err) + } + + if err := k.RunKubectlEnsureNS(ctx, "actions-runner-system", testing.KubectlConfig{Env: kubectlEnv}); err != nil { + t.Fatal(err) + } + }) + + // If you're using VS Code and wanting to run this test locally, + // Browse "Workspace Settings" and search for "go test env file" and put e.g. "${workspaceFolder}/.test.env" there + githubToken := os.Getenv("GITHUB_TOKEN") + if githubToken == "" { + t.Fatal("GITHUB_TOKEN must be set") + } + + scriptEnv := []string{ + "KUBECONFIG=" + k.Kubeconfig(), + "NAME=" + controllerImageRepo, + "VERSION=" + controllerImageTag, + "RUNNER_NAME=" + runnerImageRepo, + "RUNNER_TAG=" + runnerImageTag, + "TEST_REPO=" + "actions-runner-controller/mumoshu-actions-test", + "TEST_ORG=" + "actions-runner-controller", + "TEST_ORG_REPO=" + "actions-runner-controller/mumoshu-actions-test-org-runners", + "SYNC_PERIOD=" + "10s", + "USE_RUNNERSET=" + "1", + "ACCEPTANCE_TEST_DEPLOYMENT_TOOL=" + "helm", + "ACCEPTANCE_TEST_SECRET_TYPE=token", + "GITHUB_TOKEN=" + githubToken, + } + + t.Run("install actions-runner-controller", func(t *testing.T) { + if err := k.RunScript(ctx, "../../acceptance/deploy.sh", testing.ScriptConfig{Dir: "../..", Env: scriptEnv}); err != nil { + t.Fatal(err) + } + }) +} diff --git a/testing/testing.go b/testing/testing.go new file mode 100644 index 0000000000..0c678b6e49 --- /dev/null +++ b/testing/testing.go @@ -0,0 +1,409 @@ +package testing + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +type T = testing.T + +var Short = testing.Short + +// Cluster is a test cluster backend by a kind cluster and the dockerd powering it. +// It intracts with the kind cluster via the kind command and dockerd via the docker command +// for various operations that otherwise needs to be automated via shell scripts or makefiles. +type Cluster struct { + // Name is the name of the cluster + Name string + + // Dir is the path to the directory that contains various temporary files like a kind cluster config yaml for testing. + // This is occasionally the value returned by testing.TempDir() so that + // you don't need to clean it up yourself. + Dir string + + kubeconfig string +} + +type Config struct { + // PreloadImages is the list of container images to be pulled and loaded into the cluster. + // This might be useful to speed up your test by avoiding to let dockerd pull images from the internet each time you need to + // run tests. + PreloadImages []ContainerImage +} + +type Option = func(*Config) + +func Preload(imgs ...ContainerImage) Option { + return func(c *Config) { + c.PreloadImages = append(c.PreloadImages, imgs...) + } +} + +type ContainerImage struct { + Repo, Tag string +} + +func Start(t *testing.T, k Cluster, opts ...Option) *Cluster { + t.Helper() + + invalidChars := []string{"/"} + + name := strings.ToLower(t.Name()) + + for _, c := range invalidChars { + name = strings.ReplaceAll(name, c, "") + } + + k.Name = name + k.Dir = t.TempDir() + + kk := &k + if err := kk.Start(context.Background()); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + var run string + for i := range os.Args { + // `go test -run $RUN` results in `/tmp/path/to/some.test -test.run $RUN` being run, + // and hence we check for -test.run + if os.Args[i] == "-test.run" { + runIdx := i + 1 + run = os.Args[runIdx] + break + } else if strings.HasPrefix(os.Args[i], "-test.run=") { + split := strings.Split(os.Args[i], "-test.run=") + run = split[1] + } + } + + if t.Failed() { + return + } + + // Do not delete the cluster so that we can accelerate interation on tests + if run != "" && run != "^"+t.Name()+"$" { + // This should be printed to the debug console for visibility + t.Logf("Skipped stopping cluster due to run being %q", run) + return + } + + kk.Stop(context.Background()) + }) + + var cfg Config + + for _, o := range opts { + o(&cfg) + } + + if err := k.PullImages(context.Background(), cfg.PreloadImages); err != nil { + t.Fatal(err) + } + + if err := k.LoadImages(context.Background(), cfg.PreloadImages); err != nil { + t.Fatal(err) + } + + if err := k.writeKubeconfig(context.Background()); err != nil { + t.Fatal(err) + } + + return kk +} + +func (k *Cluster) Kubeconfig() string { + return k.kubeconfig +} + +func (k *Cluster) Start(ctx context.Context) error { + getNodes, err := k.combinedOutput(k.kindGetNodesCmd(ctx, k.Name)) + if err != nil { + return err + } + + getNodes = strings.TrimSpace(getNodes) + + if getNodes == fmt.Sprintf("No kind nodes found for cluster %q.", k.Name) { + f, err := os.CreateTemp(k.Dir, k.Name+".kind.yaml") + if err != nil { + return err + } + + kindConfig := []byte(fmt.Sprintf(`kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: %s +`, k.Name)) + + if err := os.WriteFile(f.Name(), kindConfig, 0644); err != nil { + return err + } + + if _, err := k.combinedOutput(k.kindCreateCmd(ctx, k.Name, f.Name())); err != nil { + return err + } + } + + return nil +} + +func (k *Cluster) combinedOutput(cmd *exec.Cmd) (string, error) { + o, err := cmd.CombinedOutput() + if err != nil { + args := append([]string{}, cmd.Args...) + args[0] = cmd.Path + + cs := strings.Join(args, " ") + s := string(o) + k.errorf("%s failed with output:\n%s", cs, s) + + return s, err + } + + return string(o), nil +} + +func (k *Cluster) errorf(f string, args ...interface{}) { + fmt.Fprintf(os.Stderr, f+"\n", args...) +} + +func (k *Cluster) kindGetNodesCmd(ctx context.Context, cluster string) *exec.Cmd { + return exec.CommandContext(ctx, "kind", "get", "nodes", "--name", cluster) +} + +func (k *Cluster) kindCreateCmd(ctx context.Context, cluster, configFile string) *exec.Cmd { + return exec.CommandContext(ctx, "kind", "create", "cluster", "--name", cluster, "--config", configFile) +} + +type DockerBuild struct { + Dockerfile string + Args []BuildArg + Image ContainerImage +} + +type BuildArg struct { + Name, Value string +} + +func (k *Cluster) BuildImages(ctx context.Context, builds []DockerBuild) error { + for _, build := range builds { + var args []string + args = append(args, "--build-arg=TARGETPLATFORM="+"linux/amd64") + for _, buildArg := range build.Args { + args = append(args, "--build-arg="+buildArg.Name+"="+buildArg.Value) + } + _, err := k.combinedOutput(k.dockerBuildCmd(ctx, build.Dockerfile, build.Image.Repo, build.Image.Tag, args)) + + if err != nil { + return fmt.Errorf("failed building %v: %w", build, err) + } + } + + return nil +} + +func (k *Cluster) dockerBuildCmd(ctx context.Context, dockerfile, repo, tag string, args []string) *exec.Cmd { + buildContext := filepath.Dir(dockerfile) + args = append([]string{"build", "--tag", repo + ":" + tag, "-f", dockerfile, buildContext}, args...) + + cmd := exec.CommandContext(ctx, "docker", args...) + return cmd +} + +func (k *Cluster) LoadImages(ctx context.Context, images []ContainerImage) error { + for _, img := range images { + const maxRetries = 5 + + wd, err := os.Getwd() + if err != nil { + return err + } + + tmpDir := filepath.Join(wd, ".testing", k.Name) + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return err + } + defer func() { + if tmpDir != "" && tmpDir != "/" { + os.RemoveAll(tmpDir) + } + }() + + for i := 0; i <= maxRetries; i++ { + out, err := k.combinedOutput(k.kindLoadDockerImageCmd(ctx, k.Name, img.Repo, img.Tag, tmpDir)) + + out = strings.TrimSpace(out) + + if out == fmt.Sprintf("ERROR: no nodes found for cluster %q", k.Name) { + time.Sleep(1 * time.Second) + continue + } + + if err != nil { + return fmt.Errorf("failed loading %v: %w", img, err) + } + + break + } + } + + return nil +} + +func (k *Cluster) kindLoadDockerImageCmd(ctx context.Context, cluster, repo, tag, tmpDir string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "kind", "--loglevel=trace", "load", "docker-image", repo+":"+tag, "--name", cluster) + cmd.Env = os.Environ() + // Set TMPDIR to somewhere under $HOME when you use docker installed with Ubuntu snap + // Otherwise `load docker-image` fail while running `docker save`. + // See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap + // + // In other words, it avoids errors like the below `docker save`: + // ERROR: command "docker save -o /tmp/image-tar330828066/image.tar quay.io/jetstack/cert-manager-controller:v1.1.1" failed with error: exit status 1 + // failed to save image: invalid output path: directory "/tmp/image-tar330828066" does not exist + cmd.Env = append(cmd.Env, "TMPDIR="+tmpDir) + + return cmd +} + +func (k *Cluster) PullImages(ctx context.Context, images []ContainerImage) error { + for _, img := range images { + _, err := k.combinedOutput(k.dockerPullCmd(ctx, img.Repo, img.Tag)) + if err != nil { + return err + } + } + + return nil +} + +func (k *Cluster) dockerPullCmd(ctx context.Context, repo, tag string) *exec.Cmd { + return exec.CommandContext(ctx, "docker", "pull", repo+":"+tag) +} + +func (k *Cluster) Stop(ctx context.Context) error { + if err := k.kindDeleteCmd(ctx, k.Name).Run(); err != nil { + return err + } + + return nil +} + +func (k *Cluster) kindDeleteCmd(ctx context.Context, cluster string) *exec.Cmd { + return exec.CommandContext(ctx, "kind", "delete", "cluster", "--name", cluster) +} + +func (k *Cluster) writeKubeconfig(ctx context.Context) error { + var err error + + k.kubeconfig, err = filepath.Abs(filepath.Join(k.Dir, "kubeconfig")) + if err != nil { + return err + } + + if err := k.kindExportKubeconfigCmd(ctx, k.Name, k.kubeconfig).Run(); err != nil { + return err + } + + return nil +} + +func (k *Cluster) kindExportKubeconfigCmd(ctx context.Context, cluster, path string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "kind", "export", "kubeconfig", "--name", cluster) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "KUBECONFIG="+path) + + return cmd +} + +type KubectlConfig struct { + Env []string + NoValidate bool + Timeout time.Duration + Namespace string +} + +func (k KubectlConfig) WithTimeout(o time.Duration) KubectlConfig { + k.Timeout = o + return k +} + +func (k *Cluster) RunKubectlEnsureNS(ctx context.Context, name string, cfg KubectlConfig) error { + if _, err := k.combinedOutput(k.kubectlCmd(ctx, "get", []string{"ns", name}, cfg)); err != nil { + if _, err := k.combinedOutput(k.kubectlCmd(ctx, "create", []string{"ns", name}, cfg)); err != nil { + return err + } + } + + return nil +} + +func (k *Cluster) Apply(ctx context.Context, path string, cfg KubectlConfig) error { + if _, err := k.combinedOutput(k.kubectlCmd(ctx, "apply", []string{"-f", path}, cfg)); err != nil { + return err + } + + return nil +} + +func (k *Cluster) WaitUntilDeployAvailable(ctx context.Context, name string, cfg KubectlConfig) error { + if _, err := k.combinedOutput(k.kubectlCmd(ctx, "wait", []string{"deploy/" + name, "--for=condition=available"}, cfg)); err != nil { + return err + } + + return nil +} + +func (k *Cluster) kubectlCmd(ctx context.Context, c string, args []string, cfg KubectlConfig) *exec.Cmd { + args = append([]string{c}, args...) + + if cfg.NoValidate { + args = append(args, "--validate=false") + } + + if cfg.Namespace != "" { + args = append(args, "-n="+cfg.Namespace) + } + + if cfg.Timeout > 0 { + args = append(args, "--timeout="+fmt.Sprintf("%s", cfg.Timeout)) + } + + cmd := exec.CommandContext(ctx, "kubectl", args...) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, cfg.Env...) + + return cmd +} + +type ScriptConfig struct { + Env []string + + Dir string +} + +func (k *Cluster) RunScript(ctx context.Context, path string, cfg ScriptConfig) error { + abs, err := filepath.Abs(path) + if err != nil { + return err + } + + if _, err := k.combinedOutput(k.bashRunScriptCmd(ctx, abs, cfg)); err != nil { + return err + } + + return nil +} + +func (k *Cluster) bashRunScriptCmd(ctx context.Context, path string, cfg ScriptConfig) *exec.Cmd { + cmd := exec.CommandContext(ctx, "bash", path) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, cfg.Env...) + cmd.Dir = cfg.Dir + + return cmd +}