Skip to content

Commit b0143fc

Browse files
spencerschrockspencerschrock
authored
✨ Add GitHub git compatibility mode (#4474)
* add git handler for GitHub repositories This is primarily aimed at helping in cases where a repository's .gitattributes file causes files to not be analyzed. Signed-off-by: Spencer Schrock <[email protected]> * use variadic options to configure GitHub repoclient This will let us use the new entrypoint in a backwards compatible way, similar to the scorecard.Run change made in the v5 release. Signed-off-by: Spencer Schrock <[email protected]> * add flag to enable github git mode Signed-off-by: Spencer Schrock <[email protected]> * rename flag to be forge agnostic export-ignore is not a github specific feature, and other forges, like gitlab, suffer from the same bug. Signed-off-by: Spencer Schrock <[email protected]> * move git file handler to internal package This will allow sharing with GitLab in a followup PR Signed-off-by: Spencer Schrock <[email protected]> * add a test Signed-off-by: Spencer Schrock <[email protected]> * use new toplevel gitmode argument also moves a func around for smaller PR diff. Signed-off-by: Spencer Schrock <[email protected]> * add path traversal test Signed-off-by: Spencer Schrock <[email protected]> * change flag to file-mode Signed-off-by: Spencer Schrock <[email protected]> * fix repo typo in options test the value isn't used to connect to anything though. Signed-off-by: Spencer Schrock <[email protected]> --------- Signed-off-by: Spencer Schrock <[email protected]>
1 parent 6fc296e commit b0143fc

File tree

8 files changed

+474
-23
lines changed

8 files changed

+474
-23
lines changed

‎clients/githubrepo/client.go

+92-5
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/ossf/scorecard/v5/clients"
3232
"github.com/ossf/scorecard/v5/clients/githubrepo/roundtripper"
3333
sce "github.com/ossf/scorecard/v5/errors"
34+
"github.com/ossf/scorecard/v5/internal/gitfile"
3435
"github.com/ossf/scorecard/v5/log"
3536
)
3637

@@ -40,6 +41,8 @@ var (
4041
errDefaultBranchEmpty = errors.New("default branch name is empty")
4142
)
4243

44+
type Option func(*repoClientConfig) error
45+
4346
// Client is GitHub-specific implementation of RepoClient.
4447
type Client struct {
4548
repourl *Repo
@@ -57,9 +60,32 @@ type Client struct {
5760
webhook *webhookHandler
5861
languages *languagesHandler
5962
licenses *licensesHandler
63+
git *gitfile.Handler
6064
ctx context.Context
6165
tarball tarballHandler
6266
commitDepth int
67+
gitMode bool
68+
}
69+
70+
// WithFileModeGit configures the repo client to fetch files using git.
71+
func WithFileModeGit() Option {
72+
return func(c *repoClientConfig) error {
73+
c.gitMode = true
74+
return nil
75+
}
76+
}
77+
78+
// WithRoundTripper configures the repo client to use the specified http.RoundTripper.
79+
func WithRoundTripper(rt http.RoundTripper) Option {
80+
return func(c *repoClientConfig) error {
81+
c.rt = rt
82+
return nil
83+
}
84+
}
85+
86+
type repoClientConfig struct {
87+
rt http.RoundTripper
88+
gitMode bool
6389
}
6490

6591
const defaultGhHost = "github.com"
@@ -88,8 +114,12 @@ func (client *Client) InitRepo(inputRepo clients.Repo, commitSHA string, commitD
88114
commitSHA: commitSHA,
89115
}
90116

91-
// Init tarballHandler.
92-
client.tarball.init(client.ctx, client.repo, commitSHA)
117+
if client.gitMode {
118+
client.git.Init(client.ctx, client.repo.GetCloneURL(), commitSHA)
119+
} else {
120+
// Init tarballHandler.
121+
client.tarball.init(client.ctx, client.repo, commitSHA)
122+
}
93123

94124
// Setup GraphQL.
95125
client.graphClient.init(client.ctx, client.repourl, client.commitDepth)
@@ -141,16 +171,37 @@ func (client *Client) URI() string {
141171

142172
// LocalPath implements RepoClient.LocalPath.
143173
func (client *Client) LocalPath() (string, error) {
174+
if client.gitMode {
175+
path, err := client.git.GetLocalPath()
176+
if err != nil {
177+
return "", fmt.Errorf("git local path: %w", err)
178+
}
179+
return path, nil
180+
}
144181
return client.tarball.getLocalPath()
145182
}
146183

147184
// ListFiles implements RepoClient.ListFiles.
148185
func (client *Client) ListFiles(predicate func(string) (bool, error)) ([]string, error) {
186+
if client.gitMode {
187+
files, err := client.git.ListFiles(predicate)
188+
if err != nil {
189+
return nil, fmt.Errorf("git listfiles: %w", err)
190+
}
191+
return files, nil
192+
}
149193
return client.tarball.listFiles(predicate)
150194
}
151195

152196
// GetFileReader implements RepoClient.GetFileReader.
153197
func (client *Client) GetFileReader(filename string) (io.ReadCloser, error) {
198+
if client.gitMode {
199+
f, err := client.git.GetFile(filename)
200+
if err != nil {
201+
return nil, fmt.Errorf("git getfile: %w", err)
202+
}
203+
return f, nil
204+
}
154205
return client.tarball.getFile(filename)
155206
}
156207

@@ -210,7 +261,14 @@ func (client *Client) GetOrgRepoClient(ctx context.Context) (clients.RepoClient,
210261
return nil, fmt.Errorf("error during MakeGithubRepo: %w", err)
211262
}
212263

213-
c := CreateGithubRepoClientWithTransport(ctx, client.repoClient.Client().Transport)
264+
options := []Option{WithRoundTripper(client.repoClient.Client().Transport)}
265+
if client.gitMode {
266+
options = append(options, WithFileModeGit())
267+
}
268+
c, err := NewRepoClient(ctx, options...)
269+
if err != nil {
270+
return nil, fmt.Errorf("create org repoclient: %w", err)
271+
}
214272
if err := c.InitRepo(dotGithubRepo, clients.HeadSHA, 0); err != nil {
215273
return nil, fmt.Errorf("error during InitRepo: %w", err)
216274
}
@@ -260,13 +318,40 @@ func (client *Client) SearchCommits(request clients.SearchCommitsOptions) ([]cli
260318

261319
// Close implements RepoClient.Close.
262320
func (client *Client) Close() error {
321+
if client.gitMode {
322+
if err := client.git.Cleanup(); err != nil {
323+
return fmt.Errorf("git cleanup: %w", err)
324+
}
325+
return nil
326+
}
263327
return client.tarball.cleanup()
264328
}
265329

266330
// CreateGithubRepoClientWithTransport returns a Client which implements RepoClient interface.
267331
func CreateGithubRepoClientWithTransport(ctx context.Context, rt http.RoundTripper) clients.RepoClient {
332+
//nolint:errcheck // need to suppress because this method doesn't return an error
333+
rc, _ := NewRepoClient(ctx, WithRoundTripper(rt))
334+
return rc
335+
}
336+
337+
// NewRepoClient returns a Client which implements RepoClient interface.
338+
// It can be configured with various [Option]s.
339+
func NewRepoClient(ctx context.Context, opts ...Option) (clients.RepoClient, error) {
340+
var config repoClientConfig
341+
342+
for _, option := range opts {
343+
if err := option(&config); err != nil {
344+
return nil, err
345+
}
346+
}
347+
348+
if config.rt == nil {
349+
logger := log.NewLogger(log.DefaultLevel)
350+
config.rt = roundtripper.NewTransport(ctx, logger)
351+
}
352+
268353
httpClient := &http.Client{
269-
Transport: rt,
354+
Transport: config.rt,
270355
}
271356

272357
var client *github.Client
@@ -333,7 +418,9 @@ func CreateGithubRepoClientWithTransport(ctx context.Context, rt http.RoundTripp
333418
tarball: tarballHandler{
334419
httpClient: httpClient,
335420
},
336-
}
421+
gitMode: config.gitMode,
422+
git: &gitfile.Handler{},
423+
}, nil
337424
}
338425

339426
// CreateGithubRepoClient returns a Client which implements RepoClient interface.

‎cmd/root.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -147,13 +147,18 @@ func rootCmd(o *options.Options) error {
147147
}
148148
}
149149

150-
repoResult, err = scorecard.Run(ctx, repo,
150+
opts := []scorecard.Option{
151151
scorecard.WithLogLevel(sclog.ParseLevel(o.LogLevel)),
152152
scorecard.WithCommitSHA(o.Commit),
153153
scorecard.WithCommitDepth(o.CommitDepth),
154154
scorecard.WithProbes(enabledProbes),
155155
scorecard.WithChecks(checks),
156-
)
156+
}
157+
if strings.EqualFold(o.FileMode, options.FileModeGit) {
158+
opts = append(opts, scorecard.WithFileModeGit())
159+
}
160+
161+
repoResult, err = scorecard.Run(ctx, repo, opts...)
157162
if err != nil {
158163
return fmt.Errorf("scorecard.Run: %w", err)
159164
}

‎internal/gitfile/gitfile.go

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
// Copyright 2025 OpenSSF Scorecard Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Package gitfile defines functionality to list and fetch files after temporarily cloning a git repo.
16+
package gitfile
17+
18+
import (
19+
"context"
20+
"errors"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"strings"
25+
"sync"
26+
27+
"github.com/go-git/go-git/v5"
28+
"github.com/go-git/go-git/v5/plumbing"
29+
"github.com/go-git/go-git/v5/plumbing/object"
30+
31+
"github.com/ossf/scorecard/v5/clients"
32+
)
33+
34+
var errPathTraversal = errors.New("requested file outside repo")
35+
36+
const repoDir = "repo*"
37+
38+
type Handler struct {
39+
errSetup error
40+
ctx context.Context
41+
once *sync.Once
42+
cloneURL string
43+
gitRepo *git.Repository
44+
tempDir string
45+
commitSHA string
46+
}
47+
48+
func (h *Handler) Init(ctx context.Context, cloneURL, commitSHA string) {
49+
h.errSetup = nil
50+
h.once = new(sync.Once)
51+
h.ctx = ctx
52+
h.cloneURL = cloneURL
53+
h.commitSHA = commitSHA
54+
}
55+
56+
func (h *Handler) setup() error {
57+
h.once.Do(func() {
58+
tempDir, err := os.MkdirTemp("", repoDir)
59+
if err != nil {
60+
h.errSetup = err
61+
return
62+
}
63+
h.tempDir = tempDir
64+
h.gitRepo, err = git.PlainClone(h.tempDir, false, &git.CloneOptions{
65+
URL: h.cloneURL,
66+
// TODO: auth may be required for private repos
67+
Depth: 1, // currently only use the git repo for files, dont need history
68+
SingleBranch: true,
69+
})
70+
if err != nil {
71+
h.errSetup = err
72+
return
73+
}
74+
75+
// assume the commit SHA is reachable from the default branch
76+
// this isn't as flexible as the tarball handler, but good enough for now
77+
if h.commitSHA != clients.HeadSHA {
78+
wt, err := h.gitRepo.Worktree()
79+
if err != nil {
80+
h.errSetup = err
81+
return
82+
}
83+
if err := wt.Checkout(&git.CheckoutOptions{Hash: plumbing.NewHash(h.commitSHA)}); err != nil {
84+
h.errSetup = fmt.Errorf("checkout specified commit: %w", err)
85+
return
86+
}
87+
}
88+
})
89+
return h.errSetup
90+
}
91+
92+
func (h *Handler) GetLocalPath() (string, error) {
93+
if err := h.setup(); err != nil {
94+
return "", fmt.Errorf("setup: %w", err)
95+
}
96+
return h.tempDir, nil
97+
}
98+
99+
func (h *Handler) ListFiles(predicate func(string) (bool, error)) ([]string, error) {
100+
if err := h.setup(); err != nil {
101+
return nil, fmt.Errorf("setup: %w", err)
102+
}
103+
ref, err := h.gitRepo.Head()
104+
if err != nil {
105+
return nil, fmt.Errorf("git.Head: %w", err)
106+
}
107+
108+
commit, err := h.gitRepo.CommitObject(ref.Hash())
109+
if err != nil {
110+
return nil, fmt.Errorf("git.CommitObject: %w", err)
111+
}
112+
113+
tree, err := commit.Tree()
114+
if err != nil {
115+
return nil, fmt.Errorf("git.Commit.Tree: %w", err)
116+
}
117+
118+
var files []string
119+
err = tree.Files().ForEach(func(f *object.File) error {
120+
shouldInclude, err := predicate(f.Name)
121+
if err != nil {
122+
return fmt.Errorf("error applying predicate to file %s: %w", f.Name, err)
123+
}
124+
125+
if shouldInclude {
126+
files = append(files, f.Name)
127+
}
128+
return nil
129+
})
130+
if err != nil {
131+
return nil, fmt.Errorf("git.Tree.Files: %w", err)
132+
}
133+
134+
return files, nil
135+
}
136+
137+
func (h *Handler) GetFile(filename string) (*os.File, error) {
138+
if err := h.setup(); err != nil {
139+
return nil, fmt.Errorf("setup: %w", err)
140+
}
141+
142+
// check for path traversal
143+
path := filepath.Join(h.tempDir, filename)
144+
if !strings.HasPrefix(path, filepath.Clean(h.tempDir)+string(os.PathSeparator)) {
145+
return nil, errPathTraversal
146+
}
147+
148+
f, err := os.Open(path)
149+
if err != nil {
150+
return nil, fmt.Errorf("open file: %w", err)
151+
}
152+
return f, nil
153+
}
154+
155+
func (h *Handler) Cleanup() error {
156+
if err := os.RemoveAll(h.tempDir); err != nil && !os.IsNotExist(err) {
157+
return fmt.Errorf("os.Remove: %w", err)
158+
}
159+
return nil
160+
}

0 commit comments

Comments
 (0)