Skip to content

Commit 8cba06a

Browse files
mbgmbg
authored
Merge pull request #16454 from github/mbg/go/fix/shadowed-variable
2 parents 7b89c6c + 896fb87 commit 8cba06a

File tree

5 files changed

+156
-154
lines changed

5 files changed

+156
-154
lines changed

‎go/extractor/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ go_library(
1818
"//go/extractor/dbscheme",
1919
"//go/extractor/diagnostics",
2020
"//go/extractor/srcarchive",
21+
"//go/extractor/toolchain",
2122
"//go/extractor/trap",
2223
"//go/extractor/util",
2324
"//go/extractor/vendor/golang.org/x/mod/modfile",

‎go/extractor/cli/go-autobuilder/go-autobuilder.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ func buildWithoutCustomCommands(modMode project.ModMode) bool {
333333
log.Println("Build failed, continuing to install dependencies.")
334334

335335
shouldInstallDependencies = true
336-
} else if util.DepErrors("./...", modMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...) {
336+
} else if toolchain.DepErrors("./...", modMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...) {
337337
log.Println("Dependencies are still not resolving after the build, continuing to install dependencies.")
338338

339339
shouldInstallDependencies = true

‎go/extractor/extractor.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/github/codeql-go/extractor/dbscheme"
2424
"github.com/github/codeql-go/extractor/diagnostics"
2525
"github.com/github/codeql-go/extractor/srcarchive"
26+
"github.com/github/codeql-go/extractor/toolchain"
2627
"github.com/github/codeql-go/extractor/trap"
2728
"github.com/github/codeql-go/extractor/util"
2829
"golang.org/x/tools/go/packages"
@@ -115,14 +116,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
115116
log.Println("Done extracting universe scope.")
116117

117118
// a map of package path to source directory and module root directory
118-
pkgInfos := make(map[string]util.PkgInfo)
119+
pkgInfos := make(map[string]toolchain.PkgInfo)
119120
// root directories of packages that we want to extract
120121
wantedRoots := make(map[string]bool)
121122

122123
if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "false" {
123124
log.Printf("Running go list to resolve package and module directories.")
124125
// get all packages information
125-
pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...)
126+
pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, modFlags...)
126127
if err != nil {
127128
log.Fatalf("Error getting dependency package or module directories: %v.", err)
128129
}
@@ -136,7 +137,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
136137
log.Printf("Processing package %s.", pkg.PkgPath)
137138

138139
if _, ok := pkgInfos[pkg.PkgPath]; !ok {
139-
pkgInfos[pkg.PkgPath] = util.GetPkgInfo(pkg.PkgPath, modFlags...)
140+
pkgInfos[pkg.PkgPath] = toolchain.GetPkgInfo(pkg.PkgPath, modFlags...)
140141
}
141142

142143
log.Printf("Extracting types for package %s.", pkg.PkgPath)

‎go/extractor/toolchain/toolchain.go

+150
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package toolchain
22

33
import (
44
"bufio"
5+
"encoding/json"
6+
"io"
57
"log"
68
"os"
79
"os/exec"
@@ -177,3 +179,151 @@ func Version() *exec.Cmd {
177179
version := exec.Command("go", "version")
178180
return version
179181
}
182+
183+
// Runs `go list` with `format`, `patterns`, and `flags` for the respective inputs.
184+
func RunList(format string, patterns []string, flags ...string) (string, error) {
185+
return RunListWithEnv(format, patterns, nil, flags...)
186+
}
187+
188+
// Runs `go list`.
189+
func RunListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) {
190+
args := append([]string{"list", "-e", "-f", format}, flags...)
191+
args = append(args, patterns...)
192+
cmd := exec.Command("go", args...)
193+
cmd.Env = append(os.Environ(), additionalEnv...)
194+
out, err := cmd.Output()
195+
196+
if err != nil {
197+
if exitErr, ok := err.(*exec.ExitError); ok {
198+
log.Printf("Warning: go list command failed, output below:\nstdout:\n%s\nstderr:\n%s\n", out, exitErr.Stderr)
199+
} else {
200+
log.Printf("Warning: Failed to run go list: %s", err.Error())
201+
}
202+
return "", err
203+
}
204+
205+
return strings.TrimSpace(string(out)), nil
206+
}
207+
208+
// PkgInfo holds package directory and module directory (if any) for a package
209+
type PkgInfo struct {
210+
PkgDir string // the directory directly containing source code of this package
211+
ModDir string // the module directory containing this package, empty if not a module
212+
}
213+
214+
// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
215+
// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps`
216+
// is true, all dependencies will also be included.
217+
func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
218+
// enable module mode so that we can find a module root if it exists, even if go module support is
219+
// disabled by a build
220+
if includingDeps {
221+
// the flag `-deps` causes all dependencies to be retrieved
222+
flags = append(flags, "-deps")
223+
}
224+
225+
// using -json overrides -f format
226+
output, err := RunList("", patterns, append(flags, "-json")...)
227+
if err != nil {
228+
return nil, err
229+
}
230+
231+
// the output of `go list -json` is a stream of json object
232+
type goListPkgInfo struct {
233+
ImportPath string
234+
Dir string
235+
Module *struct {
236+
Dir string
237+
}
238+
}
239+
pkgInfoMapping := make(map[string]PkgInfo)
240+
streamDecoder := json.NewDecoder(strings.NewReader(output))
241+
for {
242+
var pkgInfo goListPkgInfo
243+
decErr := streamDecoder.Decode(&pkgInfo)
244+
if decErr == io.EOF {
245+
break
246+
}
247+
if decErr != nil {
248+
log.Printf("Error decoding output of go list -json: %s", err.Error())
249+
return nil, decErr
250+
}
251+
pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
252+
if err != nil {
253+
log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
254+
}
255+
var modAbsDir string
256+
if pkgInfo.Module != nil {
257+
modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
258+
if err != nil {
259+
log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
260+
}
261+
}
262+
pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{
263+
PkgDir: pkgAbsDir,
264+
ModDir: modAbsDir,
265+
}
266+
}
267+
return pkgInfoMapping, nil
268+
}
269+
270+
// GetPkgInfo fills the package info structure for the specified package path.
271+
// It passes the `go list` the flags specified by `flags`.
272+
func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
273+
return PkgInfo{
274+
PkgDir: GetPkgDir(pkgpath, flags...),
275+
ModDir: GetModDir(pkgpath, flags...),
276+
}
277+
}
278+
279+
// GetModDir gets the absolute directory of the module containing the package with path
280+
// `pkgpath`. It passes the `go list` the flags specified by `flags`.
281+
func GetModDir(pkgpath string, flags ...string) string {
282+
// enable module mode so that we can find a module root if it exists, even if go module support is
283+
// disabled by a build
284+
mod, err := RunListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
285+
if err != nil || mod == "<nil>" {
286+
// if the command errors or modules aren't being used, return the empty string
287+
return ""
288+
}
289+
290+
modDir, err := RunListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
291+
if err != nil {
292+
return ""
293+
}
294+
295+
abs, err := filepath.Abs(modDir)
296+
if err != nil {
297+
log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error())
298+
return ""
299+
}
300+
return abs
301+
}
302+
303+
// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
304+
// `go list` command the flags specified by `flags`.
305+
func GetPkgDir(pkgpath string, flags ...string) string {
306+
pkgDir, err := RunList("{{.Dir}}", []string{pkgpath}, flags...)
307+
if err != nil {
308+
return ""
309+
}
310+
311+
abs, err := filepath.Abs(pkgDir)
312+
if err != nil {
313+
log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error())
314+
return ""
315+
}
316+
return abs
317+
}
318+
319+
// DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
320+
// list` command the flags specified by `flags`.
321+
func DepErrors(pkgpath string, flags ...string) bool {
322+
out, err := RunList("{{if .DepsErrors}}error{{else}}{{end}}", []string{pkgpath}, flags...)
323+
if err != nil {
324+
// if go list failed, assume dependencies are broken
325+
return false
326+
}
327+
328+
return out != ""
329+
}

‎go/extractor/util/util.go

-150
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package util
22

33
import (
4-
"encoding/json"
54
"errors"
6-
"io"
75
"io/fs"
86
"log"
97
"net/url"
@@ -35,154 +33,6 @@ func Getenv(key string, aliases ...string) string {
3533
return ""
3634
}
3735

38-
// runGoList is a helper function for running go list with format `format` and flags `flags` on
39-
// package `pkgpath`.
40-
func runGoList(format string, patterns []string, flags ...string) (string, error) {
41-
return runGoListWithEnv(format, patterns, nil, flags...)
42-
}
43-
44-
func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) {
45-
args := append([]string{"list", "-e", "-f", format}, flags...)
46-
args = append(args, patterns...)
47-
cmd := exec.Command("go", args...)
48-
cmd.Env = append(os.Environ(), additionalEnv...)
49-
out, err := cmd.Output()
50-
51-
if err != nil {
52-
if err, ok := err.(*exec.ExitError); ok {
53-
log.Printf("Warning: go list command failed, output below:\nstdout:\n%s\nstderr:\n%s\n", out, err.Stderr)
54-
} else {
55-
log.Printf("Warning: Failed to run go list: %s", err.Error())
56-
}
57-
return "", err
58-
}
59-
60-
return strings.TrimSpace(string(out)), nil
61-
}
62-
63-
// PkgInfo holds package directory and module directory (if any) for a package
64-
type PkgInfo struct {
65-
PkgDir string // the directory directly containing source code of this package
66-
ModDir string // the module directory containing this package, empty if not a module
67-
}
68-
69-
// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
70-
// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps`
71-
// is true, all dependencies will also be included.
72-
func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) {
73-
// enable module mode so that we can find a module root if it exists, even if go module support is
74-
// disabled by a build
75-
if includingDeps {
76-
// the flag `-deps` causes all dependencies to be retrieved
77-
flags = append(flags, "-deps")
78-
}
79-
80-
// using -json overrides -f format
81-
output, err := runGoList("", patterns, append(flags, "-json")...)
82-
if err != nil {
83-
return nil, err
84-
}
85-
86-
// the output of `go list -json` is a stream of json object
87-
type goListPkgInfo struct {
88-
ImportPath string
89-
Dir string
90-
Module *struct {
91-
Dir string
92-
}
93-
}
94-
pkgInfoMapping := make(map[string]PkgInfo)
95-
streamDecoder := json.NewDecoder(strings.NewReader(output))
96-
for {
97-
var pkgInfo goListPkgInfo
98-
decErr := streamDecoder.Decode(&pkgInfo)
99-
if decErr == io.EOF {
100-
break
101-
}
102-
if decErr != nil {
103-
log.Printf("Error decoding output of go list -json: %s", err.Error())
104-
return nil, decErr
105-
}
106-
pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
107-
if err != nil {
108-
log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
109-
}
110-
var modAbsDir string
111-
if pkgInfo.Module != nil {
112-
modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
113-
if err != nil {
114-
log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
115-
}
116-
}
117-
pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{
118-
PkgDir: pkgAbsDir,
119-
ModDir: modAbsDir,
120-
}
121-
}
122-
return pkgInfoMapping, nil
123-
}
124-
125-
// GetPkgInfo fills the package info structure for the specified package path.
126-
// It passes the `go list` the flags specified by `flags`.
127-
func GetPkgInfo(pkgpath string, flags ...string) PkgInfo {
128-
return PkgInfo{
129-
PkgDir: GetPkgDir(pkgpath, flags...),
130-
ModDir: GetModDir(pkgpath, flags...),
131-
}
132-
}
133-
134-
// GetModDir gets the absolute directory of the module containing the package with path
135-
// `pkgpath`. It passes the `go list` the flags specified by `flags`.
136-
func GetModDir(pkgpath string, flags ...string) string {
137-
// enable module mode so that we can find a module root if it exists, even if go module support is
138-
// disabled by a build
139-
mod, err := runGoListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
140-
if err != nil || mod == "<nil>" {
141-
// if the command errors or modules aren't being used, return the empty string
142-
return ""
143-
}
144-
145-
modDir, err := runGoListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...)
146-
if err != nil {
147-
return ""
148-
}
149-
150-
abs, err := filepath.Abs(modDir)
151-
if err != nil {
152-
log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error())
153-
return ""
154-
}
155-
return abs
156-
}
157-
158-
// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
159-
// `go list` command the flags specified by `flags`.
160-
func GetPkgDir(pkgpath string, flags ...string) string {
161-
pkgDir, err := runGoList("{{.Dir}}", []string{pkgpath}, flags...)
162-
if err != nil {
163-
return ""
164-
}
165-
166-
abs, err := filepath.Abs(pkgDir)
167-
if err != nil {
168-
log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error())
169-
return ""
170-
}
171-
return abs
172-
}
173-
174-
// DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
175-
// list` command the flags specified by `flags`.
176-
func DepErrors(pkgpath string, flags ...string) bool {
177-
out, err := runGoList("{{if .DepsErrors}}error{{else}}{{end}}", []string{pkgpath}, flags...)
178-
if err != nil {
179-
// if go list failed, assume dependencies are broken
180-
return false
181-
}
182-
183-
return out != ""
184-
}
185-
18636
// FileExists tests whether the file at `filename` exists and is not a directory.
18737
func FileExists(filename string) bool {
18838
info, err := os.Stat(filename)

0 commit comments

Comments
 (0)