From 2cd9bd8a43c68f65084d1fc00c9ec149df02fabb Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 4 Nov 2024 12:05:44 +0000 Subject: [PATCH 1/6] Go: Move `IsGolangVendorDirectory` to `util` package --- .../configurebaseline/configurebaseline.go | 14 +------------- go/extractor/util/util.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go/extractor/configurebaseline/configurebaseline.go b/go/extractor/configurebaseline/configurebaseline.go index 04037d61425e..062289f7baa4 100644 --- a/go/extractor/configurebaseline/configurebaseline.go +++ b/go/extractor/configurebaseline/configurebaseline.go @@ -3,24 +3,12 @@ package configurebaseline import ( "encoding/json" "io/fs" - "os" "path" "path/filepath" "github.com/github/codeql-go/extractor/util" ) -func fileExists(path string) bool { - stat, err := os.Stat(path) - return err == nil && stat.Mode().IsRegular() -} - -// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` -// and contains a `modules.txt` file. -func isGolangVendorDirectory(dirPath string) bool { - return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt")) -} - type BaselineConfig struct { PathsIgnore []string `json:"paths-ignore"` } @@ -38,7 +26,7 @@ func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) { // it will not be extracted either. return nil } - if isGolangVendorDirectory(dirPath) { + if util.IsGolangVendorDirectory(dirPath) { // Note that CodeQL expects a forward-slash-separated path, even on Windows. vendorDirs = append(vendorDirs, path.Join(filepath.ToSlash(dirPath), "**")) return filepath.SkipDir diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 595326496d8a..9a2d12988150 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -287,3 +287,15 @@ func getImportPathFromRepoURL(repourl string) string { path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "") return host + "/" + path } + +// Decides if `path` refers to a file that exists. +func fileExists(path string) bool { + stat, err := os.Stat(path) + return err == nil && stat.Mode().IsRegular() +} + +// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` +// and contains a `modules.txt` file. +func IsGolangVendorDirectory(dirPath string) bool { + return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt")) +} From b372af51b6a653d882dcac70ef82e3da8e7aae06 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 4 Nov 2024 12:16:41 +0000 Subject: [PATCH 2/6] Go: Allow `FindAllFilesWithName` to use predicate functions for `dirsToSkip` --- go/extractor/project/project.go | 8 ++++---- go/extractor/util/util.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go index 3745ca844028..d7be88475c47 100644 --- a/go/extractor/project/project.go +++ b/go/extractor/project/project.go @@ -184,12 +184,12 @@ func RemoveTemporaryExtractorFiles() { // Find all go.work files in the working directory and its subdirectories func findGoWorkFiles() []string { - return util.FindAllFilesWithName(".", "go.work", "vendor") + return util.FindAllFilesWithName(".", "go.work", util.SkipVendorChecks...) } // Find all go.mod files in the specified directory and its subdirectories func findGoModFiles(root string) []string { - return util.FindAllFilesWithName(root, "go.mod", "vendor") + return util.FindAllFilesWithName(root, "go.mod", util.SkipVendorChecks...) } // A regular expression for the Go toolchain version syntax. @@ -547,8 +547,8 @@ func startsWithAnyOf(str string, prefixes []string) bool { // Finds Go workspaces in the current working directory. func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace { bazelPaths := slices.Concat( - util.FindAllFilesWithName(".", "BUILD", "vendor"), - util.FindAllFilesWithName(".", "BUILD.bazel", "vendor"), + util.FindAllFilesWithName(".", "BUILD", util.SkipVendorChecks...), + util.FindAllFilesWithName(".", "BUILD.bazel", util.SkipVendorChecks...), ) if len(bazelPaths) > 0 { // currently not supported diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 9a2d12988150..e98e1a3d539a 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -152,7 +152,16 @@ func FindGoFiles(root string) bool { return found } -func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []string { +// The type of check function used by `FindAllFilesWithName` to decide whether to skip the directory named by `path`. +type FindAllFilesWithNameSkipCheck func(path string) bool + +// Commonly we only want to skip `vendor` directories in `FindAllFilesWithName`. This array is a suitable +// argument for `dirsToSkip` which skips `vendor` directories. +var SkipVendorChecks = []FindAllFilesWithNameSkipCheck{IsGolangVendorDirectory} + +// Returns an array of all files matching `name` within the path at `root`. +// The `dirsToSkip` array contains check functions used to decide which directories to skip. +func FindAllFilesWithName(root string, name string, dirsToSkip ...FindAllFilesWithNameSkipCheck) []string { paths := make([]string, 0, 1) filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -160,7 +169,7 @@ func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []stri } if d.IsDir() { for _, dirToSkip := range dirsToSkip { - if path == dirToSkip { + if dirToSkip(path) { return filepath.SkipDir } } From 4fa0019b5514dad044982339a5e73f1a0b5280b3 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 4 Nov 2024 12:44:42 +0000 Subject: [PATCH 3/6] Go: Add `vendor` directory to `mixed-layout` test with `go.work` file The `go.work` file here should not get discovered by the autobuilder --- .../mixed-layout/src/module/vendor/example.com/test/add.go | 5 +++++ .../mixed-layout/src/module/vendor/example.com/test/go.work | 3 +++ .../mixed-layout/src/module/vendor/modules.txt | 3 +++ .../integration-tests/mixed-layout/src/workspace/go.work.sum | 3 +++ 4 files changed, 14 insertions(+) create mode 100644 go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go create mode 100644 go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work create mode 100644 go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt create mode 100644 go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go new file mode 100644 index 000000000000..b1ce6a2a3a39 --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go @@ -0,0 +1,5 @@ +package test + +func Add(a, b int) int { + return a + b +} diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work new file mode 100644 index 000000000000..96b89a39cb9a --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work @@ -0,0 +1,3 @@ +go 1.22.0 + +use . diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt new file mode 100644 index 000000000000..023bcb386e2d --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt @@ -0,0 +1,3 @@ +# example.com/test v0.1.0 +## explicit; go 1.14 +example.com/test diff --git a/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum b/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum new file mode 100644 index 000000000000..d89ea795aab4 --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum @@ -0,0 +1,3 @@ +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= From 1d9a3dbd08bc0a2b0753068732f3cc92845c3e80 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Wed, 19 Feb 2025 12:47:28 +0000 Subject: [PATCH 4/6] Go: Make `go` happy with the `vendor` folder --- .../mixed-layout/src/module/go.mod | 2 - .../mixed-layout/src/module/go.sum | 45 ------------------- .../mixed-layout/src/module/test.go | 6 +-- .../src/module/vendor/modules.txt | 1 - 4 files changed, 1 insertion(+), 53 deletions(-) diff --git a/go/ql/integration-tests/mixed-layout/src/module/go.mod b/go/ql/integration-tests/mixed-layout/src/module/go.mod index 290a5c5c289e..c34a5e4715d6 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/go.mod +++ b/go/ql/integration-tests/mixed-layout/src/module/go.mod @@ -1,5 +1,3 @@ go 1.14 -require golang.org/x/net v0.23.0 - module module diff --git a/go/ql/integration-tests/mixed-layout/src/module/go.sum b/go/ql/integration-tests/mixed-layout/src/module/go.sum index a8e1b59ae4b1..e69de29bb2d1 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/go.sum +++ b/go/ql/integration-tests/mixed-layout/src/module/go.sum @@ -1,45 +0,0 @@ -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/ql/integration-tests/mixed-layout/src/module/test.go b/go/ql/integration-tests/mixed-layout/src/module/test.go index afc86ac3a126..502061dc123c 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/test.go +++ b/go/ql/integration-tests/mixed-layout/src/module/test.go @@ -2,12 +2,8 @@ package subdir import ( "fmt" - - "golang.org/x/net/ipv4" ) func test() { - - header := ipv4.Header{} - fmt.Print(header.String()) + fmt.Print("Hello world") } diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt index 023bcb386e2d..a7181aa2289e 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt @@ -1,3 +1,2 @@ # example.com/test v0.1.0 -## explicit; go 1.14 example.com/test From fe4ee54b6fd5c6aeb67e5b48e5c4f4dbca623aa4 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Wed, 19 Feb 2025 13:25:49 +0000 Subject: [PATCH 5/6] Go: Add more logging for `go.mod` files to workspace discovery --- go/extractor/project/project.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go index d7be88475c47..eff69c0e9cac 100644 --- a/go/extractor/project/project.go +++ b/go/extractor/project/project.go @@ -315,6 +315,11 @@ func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace { goModFiles := findGoModFiles(".") // Return a separate workspace for each `go.mod` file that we found. + if len(goModFiles) > 0 { + log.Printf("Found %d go.mod files in: %s.\n", len(goModFiles), strings.Join(goModFiles, ", ")) + } else { + log.Println("Found no go.mod files in the workspace.") + } results := make([]GoWorkspace, len(goModFiles)) for i, goModFile := range goModFiles { From 11e3a08e44bb3c722beb482d04a0d46ad3a61e74 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Wed, 19 Feb 2025 13:57:15 +0000 Subject: [PATCH 6/6] Go: Check for `modules.txt` or `glide.yaml` to exclude `vendor` dirs --- go/extractor/util/util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index e98e1a3d539a..fd3eed3e187f 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -306,5 +306,6 @@ func fileExists(path string) bool { // Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` // and contains a `modules.txt` file. func IsGolangVendorDirectory(dirPath string) bool { - return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt")) + return filepath.Base(dirPath) == "vendor" && + (fileExists(filepath.Join(dirPath, "modules.txt")) || fileExists(filepath.Join(dirPath, "../glide.yaml"))) }