Skip to content

Commit 6ad4fc0

Browse files
SkarlsoSkarlso
authored
feat: implement discovering CRDs in git repositories directly (#155)
* feat: implement discovering CRDs in git repositories directly Signed-off-by: Gergely Brautigam <[email protected]> * added a handler instead so generate crd can be used and not a new command * fix the linter Signed-off-by: Gergely Brautigam <[email protected]> * implemented git clonining on the frontend Signed-off-by: Gergely Brautigam <[email protected]> * updated the wasm version * linter Signed-off-by: Gergely Brautigam <[email protected]> * this will definitely not work when deployed Signed-off-by: Gergely Brautigam <[email protected]> * saving this for prostarity * removed the frontend feature * added README update, changed the command line option and added release doc Signed-off-by: Gergely Brautigam <[email protected]> * fixed the README sample Signed-off-by: Gergely Brautigam <[email protected]> --------- Signed-off-by: Gergely Brautigam <[email protected]>
1 parent 2a0bdf6 commit 6ad4fc0

16 files changed

+453
-117
lines changed

‎README.md

+29
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,35 @@ A single file will be created containing all versions in the CRD delimited by `-
5858

5959
Optionally, you can provide the flag `-s` which will output the generated content to `stdout`.
6060

61+
You can also point at a git repository to _discover_ CRDs inside the repository. Simply call `crd` with:
62+
63+
```
64+
➜ cty generate crd -g https://github.com/Skarlso/crd-bootstrap
65+
Discovered number of CRDs: 1
66+
```
67+
68+
The following authentication methods are available:
69+
- username and password
70+
```
71+
./cty generate crd -g https://github.com/Skarlso/crd-bootstrap --username skarlso --password password
72+
```
73+
- token
74+
```
75+
./cty generate crd -g https://github.com/Skarlso/crd-bootstrap --token token
76+
```
77+
- SSH with provided private key
78+
```
79+
./cty generate crd -g [email protected]:Skarlso/crd-bootstrap --private-ssh-key-file ~/.ssh/main-key
80+
```
81+
- SSH by using the local ssh-agent
82+
```
83+
./cty generate crd -g [email protected]:Skarlso/crd-bootstrap --ssh-agent
84+
```
85+
86+
Notice the URL change in case SSH authentication is provided.
87+
88+
Further certificate bundles can be provided for privately hosted git servers with `--ca-bundle-file`.
89+
6190
### HTML output
6291

6392
It's possible to generate a pre-rendered HTML based output for self-hosting what the website produces online.

‎cmd/crd.go

+11
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,17 @@ func constructHandler(args *rootArgs) (Handler, error) {
134134
crdHandler = &FolderHandler{location: args.folderLocation}
135135
case args.configFileLocation != "":
136136
crdHandler = &ConfigHandler{configFileLocation: args.configFileLocation}
137+
case args.gitURL != "":
138+
crdHandler = &GitHandler{
139+
URL: args.gitURL,
140+
Username: args.username,
141+
Password: args.password,
142+
Token: args.token,
143+
Tag: args.tag,
144+
caBundle: args.caBundle,
145+
privSSHKey: args.privSSHKey,
146+
useSSHAgent: args.useSSHAgent,
147+
}
137148
case args.url != "":
138149
crdHandler = &URLHandler{
139150
url: args.url,

‎cmd/generate.go

+10
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ type rootArgs struct {
1212
username string
1313
password string
1414
token string
15+
tag string
16+
caBundle string
17+
privSSHKey string
18+
useSSHAgent bool
19+
gitURL string
1520
}
1621

1722
var (
@@ -31,8 +36,13 @@ func init() {
3136
f.StringVarP(&args.fileLocation, "crd", "c", "", "The CRD file to generate a yaml from.")
3237
f.StringVarP(&args.folderLocation, "folder", "r", "", "A folder from which to parse a series of CRDs.")
3338
f.StringVarP(&args.url, "url", "u", "", "If provided, will use this URL to fetch CRD YAML content from.")
39+
f.StringVarP(&args.gitURL, "git-url", "g", "", "If provided, CRDs will be discovered using a git repository.")
3440
f.StringVar(&args.username, "username", "", "Optional username to authenticate a URL.")
3541
f.StringVar(&args.password, "password", "", "Optional password to authenticate a URL.")
3642
f.StringVar(&args.token, "token", "", "A bearer token to authenticate a URL.")
3743
f.StringVar(&args.configFileLocation, "config", "", "An optional configuration file that can define grouping data for various rendered crds.")
44+
f.StringVar(&args.tag, "tag", "", "The ref to check out. Default is head.")
45+
f.StringVar(&args.caBundle, "ca-bundle-file", "", "Additional certificate bundle to load. Should the name of the file.")
46+
f.StringVar(&args.privSSHKey, "private-ssh-key-file", "", "Private key to use for cloning. Should the name of the file.")
47+
f.BoolVar(&args.useSSHAgent, "ssh-agent", false, "If set, the configured SSH agent will be used to clone the repository..")
3848
}

‎cmd/git_handler.go

+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
9+
"github.com/go-git/go-git/v5"
10+
"github.com/go-git/go-git/v5/plumbing"
11+
"github.com/go-git/go-git/v5/plumbing/object"
12+
"github.com/go-git/go-git/v5/plumbing/transport/http"
13+
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
14+
"github.com/go-git/go-git/v5/storage/memory"
15+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
16+
"k8s.io/apimachinery/pkg/util/yaml"
17+
18+
"github.com/Skarlso/crd-to-sample-yaml/pkg"
19+
"github.com/Skarlso/crd-to-sample-yaml/pkg/sanitize"
20+
)
21+
22+
type GitHandler struct {
23+
URL string
24+
Username string
25+
Password string
26+
Token string
27+
Tag string
28+
29+
caBundle string
30+
privSSHKey string
31+
useSSHAgent bool
32+
}
33+
34+
func (g *GitHandler) CRDs() ([]*pkg.SchemaType, error) {
35+
opts, err := g.constructGitOptions()
36+
if err != nil {
37+
return nil, err
38+
}
39+
40+
r, err := git.Clone(memory.NewStorage(), nil, opts)
41+
if err != nil {
42+
return nil, fmt.Errorf("error cloning git repository: %w", err)
43+
}
44+
45+
var ref *plumbing.Reference
46+
if g.Tag != "" {
47+
ref, err = r.Tag(g.Tag)
48+
} else {
49+
ref, err = r.Head()
50+
}
51+
if err != nil {
52+
return nil, fmt.Errorf("failed to construct reference: %w", err)
53+
}
54+
55+
crds, err := gatherSchemaTypesForRef(r, ref)
56+
if err != nil {
57+
return nil, err
58+
}
59+
60+
_, _ = fmt.Fprintln(os.Stderr, "Discovered number of CRDs: ", len(crds))
61+
62+
return crds, nil
63+
}
64+
65+
func gatherSchemaTypesForRef(r *git.Repository, ref *plumbing.Reference) ([]*pkg.SchemaType, error) {
66+
// Need to resolve the ref first to the right hash otherwise it's not found.
67+
hash, err := r.ResolveRevision(plumbing.Revision(ref.Hash().String()))
68+
if err != nil {
69+
return nil, fmt.Errorf("failed to resolve revision: %w", err)
70+
}
71+
72+
commit, err := r.CommitObject(*hash)
73+
if err != nil {
74+
return nil, fmt.Errorf("error getting commit object: %w", err)
75+
}
76+
77+
commitTree, err := commit.Tree()
78+
if err != nil {
79+
return nil, err
80+
}
81+
82+
var crds []*pkg.SchemaType
83+
// Tried to make this concurrent, but there was very little gain. It just takes this long to
84+
// clone a large repository. It's not the processing OR the rendering that takes long.
85+
if err := commitTree.Files().ForEach(func(f *object.File) error {
86+
crd, err := processEntry(f)
87+
if err != nil {
88+
return err
89+
}
90+
91+
if crd != nil {
92+
crds = append(crds, crd)
93+
}
94+
95+
return nil
96+
}); err != nil {
97+
return nil, err
98+
}
99+
100+
return crds, nil
101+
}
102+
103+
func processEntry(f *object.File) (*pkg.SchemaType, error) {
104+
for _, path := range strings.Split(f.Name, string(filepath.Separator)) {
105+
if path == "test" {
106+
return nil, nil
107+
}
108+
}
109+
110+
if ext := filepath.Ext(f.Name); ext != ".yaml" {
111+
return nil, nil
112+
}
113+
114+
content, err := f.Contents()
115+
if err != nil {
116+
return nil, err
117+
}
118+
119+
sanitized, err := sanitize.Sanitize([]byte(content))
120+
if err != nil {
121+
return nil, fmt.Errorf("failed to sanitize content: %w", err)
122+
}
123+
124+
crd := &unstructured.Unstructured{}
125+
if err := yaml.Unmarshal(sanitized, crd); err != nil {
126+
return nil, nil //nolint:nilerr // intentional
127+
}
128+
129+
schemaType, err := pkg.ExtractSchemaType(crd)
130+
if err != nil {
131+
return nil, nil //nolint:nilerr // intentional
132+
}
133+
134+
return schemaType, nil
135+
}
136+
137+
func (g *GitHandler) constructGitOptions() (*git.CloneOptions, error) {
138+
opts := &git.CloneOptions{
139+
URL: g.URL,
140+
Depth: 1,
141+
}
142+
143+
// trickle down. if ssh key is set, this will be overwritten.
144+
if g.Username != "" && g.Password != "" {
145+
opts.Auth = &http.BasicAuth{
146+
Username: g.Username,
147+
Password: g.Password,
148+
}
149+
}
150+
if g.Token != "" {
151+
opts.Auth = &http.TokenAuth{
152+
Token: g.Token,
153+
}
154+
}
155+
if g.caBundle != "" {
156+
opts.CABundle = []byte(g.caBundle)
157+
}
158+
if g.privSSHKey != "" {
159+
if !strings.Contains(g.URL, "@") {
160+
return nil, fmt.Errorf("git URL does not contain an ssh address: %s", g.URL)
161+
}
162+
163+
keys, err := ssh.NewPublicKeysFromFile("git", g.privSSHKey, g.Password)
164+
if err != nil {
165+
return nil, err
166+
}
167+
168+
opts.Auth = keys
169+
}
170+
if g.useSSHAgent {
171+
if !strings.Contains(g.URL, "@") {
172+
return nil, fmt.Errorf("git URL does not contain an ssh address: %s", g.URL)
173+
}
174+
175+
authMethod, err := ssh.NewSSHAgentAuth("git")
176+
if err != nil {
177+
return nil, err
178+
}
179+
opts.Auth = authMethod
180+
}
181+
182+
return opts, nil
183+
}

‎docs/release_notes/v1.1.1.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Release v1.1.1
2+
3+
- feat: implement discovering CRDs in git repositories directly #155
4+
5+
This feature adds the ability to fetch CRDs through discovery from a git based repository.
6+
It is only supported through the CLI at the moment. For more information, check out the
7+
Pull Request description and the README.

‎go.mod

+24-3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
github.com/brianvoe/gofakeit/v6 v6.28.0
99
github.com/fatih/color v1.18.0
1010
github.com/fxamacker/cbor/v2 v2.7.0
11+
github.com/go-git/go-git/v5 v5.12.0
1112
github.com/google/go-cmp v0.6.0
1213
github.com/jedib0t/go-pretty/v6 v6.6.5
1314
github.com/maxence-charriere/go-app/v10 v10.0.9
@@ -19,57 +20,77 @@ require (
1920

2021
require (
2122
cel.dev/expr v0.18.0 // indirect
23+
dario.cat/mergo v1.0.1 // indirect
24+
github.com/Microsoft/go-winio v0.6.2 // indirect
25+
github.com/ProtonMail/go-crypto v1.1.3 // indirect
2226
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
2327
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
2428
github.com/beorn7/perks v1.0.1 // indirect
2529
github.com/blang/semver/v4 v4.0.0 // indirect
2630
github.com/cespare/xxhash/v2 v2.3.0 // indirect
31+
github.com/cloudflare/circl v1.5.0 // indirect
32+
github.com/cyphar/filepath-securejoin v0.3.5 // indirect
2733
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
34+
github.com/emirpasic/gods v1.18.1 // indirect
35+
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
36+
github.com/go-git/go-billy/v5 v5.6.0 // indirect
2837
github.com/go-logr/logr v1.4.2 // indirect
2938
github.com/go-openapi/jsonpointer v0.21.0 // indirect
3039
github.com/go-openapi/jsonreference v0.20.2 // indirect
3140
github.com/go-openapi/swag v0.23.0 // indirect
3241
github.com/gogo/protobuf v1.3.2 // indirect
42+
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
3343
github.com/golang/protobuf v1.5.4 // indirect
3444
github.com/google/cel-go v0.22.0 // indirect
3545
github.com/google/gnostic-models v0.6.8 // indirect
3646
github.com/google/gofuzz v1.2.0 // indirect
3747
github.com/google/uuid v1.6.0 // indirect
3848
github.com/inconshreveable/mousetrap v1.1.0 // indirect
49+
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
3950
github.com/josharian/intern v1.0.0 // indirect
4051
github.com/json-iterator/go v1.1.12 // indirect
52+
github.com/kevinburke/ssh_config v1.2.0 // indirect
4153
github.com/mailru/easyjson v0.7.7 // indirect
4254
github.com/mattn/go-colorable v0.1.13 // indirect
4355
github.com/mattn/go-isatty v0.0.20 // indirect
4456
github.com/mattn/go-runewidth v0.0.15 // indirect
4557
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
4658
github.com/modern-go/reflect2 v1.0.2 // indirect
4759
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
60+
github.com/onsi/gomega v1.36.1 // indirect
61+
github.com/pjbgf/sha1cd v0.3.0 // indirect
4862
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
4963
github.com/prometheus/client_golang v1.19.1 // indirect
5064
github.com/prometheus/client_model v0.6.1 // indirect
5165
github.com/prometheus/common v0.55.0 // indirect
5266
github.com/prometheus/procfs v0.15.1 // indirect
5367
github.com/rivo/uniseg v0.2.0 // indirect
68+
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
69+
github.com/skeema/knownhosts v1.3.0 // indirect
5470
github.com/spf13/pflag v1.0.5 // indirect
5571
github.com/stoewer/go-strcase v1.3.0 // indirect
5672
github.com/x448/float16 v0.8.4 // indirect
73+
github.com/xanzy/ssh-agent v0.3.3 // indirect
5774
go.opentelemetry.io/otel v1.28.0 // indirect
5875
go.opentelemetry.io/otel/trace v1.28.0 // indirect
76+
golang.org/x/crypto v0.30.0 // indirect
5977
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
60-
golang.org/x/net v0.30.0 // indirect
61-
golang.org/x/sys v0.26.0 // indirect
78+
golang.org/x/net v0.32.0 // indirect
79+
golang.org/x/oauth2 v0.24.0 // indirect
80+
golang.org/x/sys v0.28.0 // indirect
6281
golang.org/x/text v0.21.0 // indirect
82+
golang.org/x/time v0.8.0 // indirect
6383
google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
6484
google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
6585
google.golang.org/protobuf v1.35.1 // indirect
6686
gopkg.in/inf.v0 v0.9.1 // indirect
87+
gopkg.in/warnings.v0 v0.1.2 // indirect
6788
gopkg.in/yaml.v3 v3.0.1 // indirect
6889
k8s.io/apiserver v0.32.0 // indirect
6990
k8s.io/component-base v0.32.0 // indirect
7091
k8s.io/klog/v2 v2.130.1 // indirect
7192
k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
72-
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
93+
k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 // indirect
7394
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
7495
sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
7596
sigs.k8s.io/yaml v1.4.0 // indirect

0 commit comments

Comments
 (0)