Unattended Flux installation for GitHub repos (automatic add of deploy key)

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/gofrs/flock v0.7.1
 	github.com/golangci/golangci-lint v1.27.0
+	github.com/google/go-github/v31 v31.0.0
 	github.com/goreleaser/goreleaser v0.136.0
 	github.com/instrumenta/kubeval v0.0.0-20190918223246-8d013ec9fc56
 	github.com/justinbarrick/go-k8s-portforward v1.0.3
@@ -48,6 +49,7 @@ require (
 	github.com/weaveworks/github-release v0.6.3-0.20161024133933-73deea6af1e8
 	github.com/weaveworks/launcher v0.0.0-20180711153254-f1b2830d4f2d
 	github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/sys v0.0.0-20200428200454-593003d681fa // indirect
 	golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770
 	k8s.io/api v0.16.8

go.sum

@@ -449,8 +449,12 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-github/v25 v25.0.1 h1:s405kPD52lKa1MVxiEumod/E6/+0pvQ8Ed/sT65DjKc=
+github.com/google/go-github/v25 v25.0.1/go.mod h1:6z5pC69qHtrPJ0sXPsj4BLnd82b+r6sLB7qcBoRZqpw=
 github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo=
 github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
+github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo=
+github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic=

pkg/apis/eksctl.io/v1alpha5/assets/schema.json

@@ -816,6 +816,9 @@
     },
     "Operator": {
       "properties": {
+        "commitOperatorManifests": {
+          "type": "boolean"
+        },
         "label": {
           "type": "string"
         },
@@ -824,6 +827,9 @@
         },
         "withHelm": {
           "type": "boolean"
+        },
+        "readOnly": {
+          "type": "boolean"
         }
       },
       "additionalProperties": false,

pkg/apis/eksctl.io/v1alpha5/defaults.go

@@ -248,6 +248,10 @@ func SetDefaultGitSettings(c *ClusterConfig) {
 		return
 	}
 
+	if c.Git.Operator.CommitOperatorManifests == nil {
+		c.Git.Operator.CommitOperatorManifests = Enabled()
+	}
+
 	if c.Git.Operator.Label == "" {
 		c.Git.Operator.Label = "flux"
 	}

pkg/apis/eksctl.io/v1alpha5/types.go

@@ -743,12 +743,16 @@ type Repo struct {
 // Operator groups all configuration options related to the operator used to
 // keep the cluster and the Git repository in sync.
 type Operator struct {
+	// +optional
+	CommitOperatorManifests *bool `json:"commitOperatorManifests,omitempty"` // Commit and push Flux manifests to the Git Repo on install
 	// +optional
 	Label string `json:"label,omitempty"` // e.g. flux
 	// +optional
 	Namespace string `json:"namespace,omitempty"` // e.g. flux
 	// +optional
 	WithHelm *bool `json:"withHelm,omitempty"` // whether to install the Flux Helm Operator or not
+	// +optional
+	ReadOnly bool `json:"readOnly,omitempty"` // Instruct Flux to read-only mode and create the deploy key as read-only
 }
 
 // Profile groups all details on a quickstart profile to enable on the cluster

pkg/ctl/cmdutils/gitops.go

@@ -23,8 +23,11 @@ const (
 	gitFluxPath = "git-flux-subdir"
 	gitLabel    = "git-label"
 	namespace   = "namespace"
+	readOnly    = "read-only"
 	withHelm    = "with-helm"
 
+	commitOperatorManifests = "commit-operator-manifests"
+
 	profileName     = "profile-source"
 	profileRevision = "profile-revision"
 )
@@ -51,6 +54,10 @@ func AddCommonFlagsForFlux(fs *pflag.FlagSet, opts *api.Git) {
 		"Directory within the Git repository where to commit the Flux manifests")
 	fs.StringVar(&opts.Operator.Namespace, namespace, "flux",
 		"Cluster namespace where to install Flux, the Helm Operator and Tiller")
+	fs.BoolVar(&opts.Operator.ReadOnly, readOnly, false,
+		"Instruct Flux to read-only mode and create the deploy key as read-only")
+	opts.Operator.CommitOperatorManifests = fs.Bool(commitOperatorManifests, true,
+		"Commit and push Flux manifests to the Git Repo on install")
 	opts.Operator.WithHelm = fs.Bool(withHelm, true, "Install the Helm Operator and Tiller")
 }
 
@@ -65,7 +72,8 @@ func AddCommonFlagsForGit(fs *pflag.FlagSet, repo *api.Repo) {
 		"Username to use as Git committer")
 	fs.StringVar(&repo.Email, gitEmail, "",
 		"Email to use as Git committer")
-	fs.StringVar(&repo.PrivateSSHKeyPath, gitPrivateSSHKeyPath, "",
+	fs.StringVar(&repo.PrivateSSHKeyPath,
+		gitPrivateSSHKeyPath, "",
 		"Optional path to the private SSH key to use with Git, e.g. ~/.ssh/id_rsa")
 }
 

pkg/ctl/delete/cluster.go

@@ -16,6 +16,7 @@ import (
 	"github.com/weaveworks/eksctl/pkg/cfn/manager"
 	"github.com/weaveworks/eksctl/pkg/ctl/cmdutils"
 	"github.com/weaveworks/eksctl/pkg/elb"
+	"github.com/weaveworks/eksctl/pkg/gitops/deploykey"
 	iamoidc "github.com/weaveworks/eksctl/pkg/iam/oidc"
 	"github.com/weaveworks/eksctl/pkg/kubernetes"
 	"github.com/weaveworks/eksctl/pkg/printers"
@@ -184,6 +185,12 @@ func doDeleteCluster(cmd *cmdutils.Cmd) error {
 		logger.Success("all cluster resources were deleted")
 	}
 
+	{
+		if err := deploykey.Delete(context.Background(), cfg); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

pkg/ctl/enable/repo.go

@@ -93,6 +93,8 @@ func doEnableRepository(cmd *cmdutils.Cmd) error {
 		logger.Critical("unable to set up gitops repo: %s", err.Error())
 		return err
 	}
+
 	logger.Info(userInstructions)
-	return err
+
+	return nil
 }

pkg/gitops/deploykey/deploykey.go

@@ -0,0 +1,74 @@
+package deploykey
+
+import (
+	"context"
+	"os"
+
+	"github.com/kris-nova/logger"
+	"github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+)
+
+type GitProvider interface {
+	Put(ctx context.Context, fluxSSHKey PublicKey) error
+	Delete(ctx context.Context) error
+}
+
+func ForCluster(cluster *v1alpha5.ClusterConfig) GitProvider {
+	var (
+		repoURL  string
+		readOnly bool
+	)
+
+	if git := cluster.Git; git != nil {
+		if repo := git.Repo; repo != nil {
+			repoURL = repo.URL
+		}
+
+		readOnly = git.Operator.ReadOnly
+	}
+
+	if repoURL == "" {
+		return nil
+	}
+
+	if owner, repo, ok := getGitHubOwnerRepoFromRepoURL(repoURL); !ok {
+		logger.Info("skipped managing GitHub deploy key for URL %s: Only `[email protected]:OWNER/REPO.git` is accepted for automatic deploy key creation", repoURL)
+	} else if githubToken := os.Getenv(EnvVarGitHubToken); githubToken == "" {
+		logger.Info("GITHUB_TOKEN is not set. Please set it so that eksctl is able to create and delete GitHub deploy key from Flux SSH public key")
+	} else {
+		return &GitHubProvider{
+			cluster:     cluster.Metadata,
+			githubToken: githubToken,
+			readOnly:    readOnly,
+			owner:       owner,
+			repo:        repo,
+		}
+	}
+
+	return nil
+}
+
+func Put(ctx context.Context, cluster *v1alpha5.ClusterConfig, fluxSSHKey PublicKey) (bool, error) {
+	p := ForCluster(cluster)
+
+	if p == nil {
+		return false, nil
+	}
+
+	return true, p.Put(ctx, fluxSSHKey)
+}
+
+func Delete(ctx context.Context, cluster *v1alpha5.ClusterConfig) error {
+	p := ForCluster(cluster)
+
+	if p == nil {
+		return nil
+	}
+
+	return p.Delete(ctx)
+}
+
+// PublicKey represents a public SSH key as it is returned by flux
+type PublicKey struct {
+	Key string
+}

pkg/gitops/deploykey/github.go

@@ -0,0 +1,119 @@
+package deploykey
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/google/go-github/v31/github"
+	"github.com/kris-nova/logger"
+	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+	"golang.org/x/oauth2"
+)
+
+const (
+	EnvVarGitHubToken = "GITHUB_TOKEN"
+)
+
+type GitHubProvider struct {
+	cluster     *api.ClusterMeta
+	owner, repo string
+	readOnly    bool
+	githubToken string
+}
+
+func (p *GitHubProvider) Put(ctx context.Context, fluxSSHKey PublicKey) error {
+	gh := p.getGitHubAPIClient(ctx)
+
+	logger.Info("creating GitHub deploy key from Flux SSH public key")
+
+	title := p.getDeployKeyTitle()
+
+	key, _, err := gh.Repositories.CreateKey(ctx, p.owner, p.repo, &github.Key{
+		Key:      &fluxSSHKey.Key,
+		Title:    &title,
+		ReadOnly: &p.readOnly,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	logger.Info("%s configured with Flux SSH public key\n%s", *key.Title, fluxSSHKey.Key)
+
+	return nil
+}
+
+func (p *GitHubProvider) Delete(ctx context.Context) error {
+	gh := p.getGitHubAPIClient(ctx)
+
+	logger.Info("deleting GitHub deploy key")
+
+	title := p.getDeployKeyTitle()
+
+	keys, _, err := gh.Repositories.ListKeys(ctx, p.owner, p.repo, &github.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	var keyID int64
+
+	for _, key := range keys {
+		if key.GetTitle() == title {
+			keyID = key.GetID()
+
+			break
+		}
+	}
+
+	if keyID == 0 {
+		logger.Info("skipped deleting GitHub deploy key %q: The key does not exist. Probably you've already deleted it?")
+
+		return nil
+	}
+
+	if _, err := gh.Repositories.DeleteKey(ctx, p.owner, p.repo, keyID); err != nil {
+		return err
+	}
+
+	logger.Info("deleted GitHub deploy key %s", title)
+
+	return nil
+}
+
+func (p *GitHubProvider) getGitHubAPIClient(ctx context.Context) *github.Client {
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: p.githubToken},
+	)
+	tc := oauth2.NewClient(ctx, ts)
+	gh := github.NewClient(tc)
+
+	return gh
+}
+
+func getGitHubOwnerRepoFromRepoURL(repoURL string) (string, string, bool) {
+	if repoURL == "" {
+		return "", "", false
+	}
+
+	sshFull := regexp.MustCompile(`ssh://[email protected]/([^/]+)/([^.]+).git`)
+	sshShort := regexp.MustCompile(`[email protected]:([^/]+)/([^.]+).git`)
+
+	patterns := []*regexp.Regexp{
+		sshFull,
+		sshShort,
+	}
+
+	for _, p := range patterns {
+		m := p.FindStringSubmatch(repoURL)
+		if len(m) == 3 {
+			return m[1], m[2], true
+		}
+	}
+
+	return "", "", false
+}
+
+func (p *GitHubProvider) getDeployKeyTitle() string {
+	return fmt.Sprintf("eksctl-flux-%s-%s", p.cluster.Region, p.cluster.Name)
+}