diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 7e576927f662..8c69d9cc01ff 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "//go/extractor/dbscheme", "//go/extractor/diagnostics", "//go/extractor/srcarchive", + "//go/extractor/toolchain", "//go/extractor/trap", "//go/extractor/util", "//go/extractor/vendor/golang.org/x/mod/modfile", diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 2e9731c989b8..a4d01b29c5d7 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -333,7 +333,7 @@ func buildWithoutCustomCommands(modMode project.ModMode) bool { log.Println("Build failed, continuing to install dependencies.") shouldInstallDependencies = true - } else if util.DepErrors("./...", modMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...) { + } else if toolchain.DepErrors("./...", modMode.ArgsForGoVersion(toolchain.GetEnvGoSemVer())...) { log.Println("Dependencies are still not resolving after the build, continuing to install dependencies.") shouldInstallDependencies = true diff --git a/go/extractor/extractor.go b/go/extractor/extractor.go index d7759326d57a..090bd486c3a7 100644 --- a/go/extractor/extractor.go +++ b/go/extractor/extractor.go @@ -23,6 +23,7 @@ import ( "github.com/github/codeql-go/extractor/dbscheme" "github.com/github/codeql-go/extractor/diagnostics" "github.com/github/codeql-go/extractor/srcarchive" + "github.com/github/codeql-go/extractor/toolchain" "github.com/github/codeql-go/extractor/trap" "github.com/github/codeql-go/extractor/util" "golang.org/x/tools/go/packages" @@ -115,14 +116,14 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { log.Println("Done extracting universe scope.") // a map of package path to source directory and module root directory - pkgInfos := make(map[string]util.PkgInfo) + pkgInfos := make(map[string]toolchain.PkgInfo) // root directories of packages that we want to extract wantedRoots := make(map[string]bool) if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "false" { log.Printf("Running go list to resolve package and module directories.") // get all packages information - pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...) + pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, modFlags...) if err != nil { log.Fatalf("Error getting dependency package or module directories: %v.", err) } @@ -136,7 +137,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error { log.Printf("Processing package %s.", pkg.PkgPath) if _, ok := pkgInfos[pkg.PkgPath]; !ok { - pkgInfos[pkg.PkgPath] = util.GetPkgInfo(pkg.PkgPath, modFlags...) + pkgInfos[pkg.PkgPath] = toolchain.GetPkgInfo(pkg.PkgPath, modFlags...) } log.Printf("Extracting types for package %s.", pkg.PkgPath) diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 01befa4384ea..89ad1e35ecc3 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -2,6 +2,8 @@ package toolchain import ( "bufio" + "encoding/json" + "io" "log" "os" "os/exec" @@ -177,3 +179,151 @@ func Version() *exec.Cmd { version := exec.Command("go", "version") return version } + +// Runs `go list` with `format`, `patterns`, and `flags` for the respective inputs. +func RunList(format string, patterns []string, flags ...string) (string, error) { + return RunListWithEnv(format, patterns, nil, flags...) +} + +// Runs `go list`. +func RunListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) { + args := append([]string{"list", "-e", "-f", format}, flags...) + args = append(args, patterns...) + cmd := exec.Command("go", args...) + cmd.Env = append(os.Environ(), additionalEnv...) + out, err := cmd.Output() + + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + log.Printf("Warning: go list command failed, output below:\nstdout:\n%s\nstderr:\n%s\n", out, exitErr.Stderr) + } else { + log.Printf("Warning: Failed to run go list: %s", err.Error()) + } + return "", err + } + + return strings.TrimSpace(string(out)), nil +} + +// PkgInfo holds package directory and module directory (if any) for a package +type PkgInfo struct { + PkgDir string // the directory directly containing source code of this package + ModDir string // the module directory containing this package, empty if not a module +} + +// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the +// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps` +// is true, all dependencies will also be included. +func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) { + // enable module mode so that we can find a module root if it exists, even if go module support is + // disabled by a build + if includingDeps { + // the flag `-deps` causes all dependencies to be retrieved + flags = append(flags, "-deps") + } + + // using -json overrides -f format + output, err := RunList("", patterns, append(flags, "-json")...) + if err != nil { + return nil, err + } + + // the output of `go list -json` is a stream of json object + type goListPkgInfo struct { + ImportPath string + Dir string + Module *struct { + Dir string + } + } + pkgInfoMapping := make(map[string]PkgInfo) + streamDecoder := json.NewDecoder(strings.NewReader(output)) + for { + var pkgInfo goListPkgInfo + decErr := streamDecoder.Decode(&pkgInfo) + if decErr == io.EOF { + break + } + if decErr != nil { + log.Printf("Error decoding output of go list -json: %s", err.Error()) + return nil, decErr + } + pkgAbsDir, err := filepath.Abs(pkgInfo.Dir) + if err != nil { + log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error()) + } + var modAbsDir string + if pkgInfo.Module != nil { + modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir) + if err != nil { + log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error()) + } + } + pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{ + PkgDir: pkgAbsDir, + ModDir: modAbsDir, + } + } + return pkgInfoMapping, nil +} + +// GetPkgInfo fills the package info structure for the specified package path. +// It passes the `go list` the flags specified by `flags`. +func GetPkgInfo(pkgpath string, flags ...string) PkgInfo { + return PkgInfo{ + PkgDir: GetPkgDir(pkgpath, flags...), + ModDir: GetModDir(pkgpath, flags...), + } +} + +// GetModDir gets the absolute directory of the module containing the package with path +// `pkgpath`. It passes the `go list` the flags specified by `flags`. +func GetModDir(pkgpath string, flags ...string) string { + // enable module mode so that we can find a module root if it exists, even if go module support is + // disabled by a build + mod, err := RunListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...) + if err != nil || mod == "" { + // if the command errors or modules aren't being used, return the empty string + return "" + } + + modDir, err := RunListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...) + if err != nil { + return "" + } + + abs, err := filepath.Abs(modDir) + if err != nil { + log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error()) + return "" + } + return abs +} + +// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the +// `go list` command the flags specified by `flags`. +func GetPkgDir(pkgpath string, flags ...string) string { + pkgDir, err := RunList("{{.Dir}}", []string{pkgpath}, flags...) + if err != nil { + return "" + } + + abs, err := filepath.Abs(pkgDir) + if err != nil { + log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error()) + return "" + } + return abs +} + +// DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go +// list` command the flags specified by `flags`. +func DepErrors(pkgpath string, flags ...string) bool { + out, err := RunList("{{if .DepsErrors}}error{{else}}{{end}}", []string{pkgpath}, flags...) + if err != nil { + // if go list failed, assume dependencies are broken + return false + } + + return out != "" +} diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 2ae6a2b0cd2b..efde5988193c 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -1,9 +1,7 @@ package util import ( - "encoding/json" "errors" - "io" "io/fs" "log" "net/url" @@ -35,154 +33,6 @@ func Getenv(key string, aliases ...string) string { return "" } -// runGoList is a helper function for running go list with format `format` and flags `flags` on -// package `pkgpath`. -func runGoList(format string, patterns []string, flags ...string) (string, error) { - return runGoListWithEnv(format, patterns, nil, flags...) -} - -func runGoListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) (string, error) { - args := append([]string{"list", "-e", "-f", format}, flags...) - args = append(args, patterns...) - cmd := exec.Command("go", args...) - cmd.Env = append(os.Environ(), additionalEnv...) - out, err := cmd.Output() - - if err != nil { - if err, ok := err.(*exec.ExitError); ok { - log.Printf("Warning: go list command failed, output below:\nstdout:\n%s\nstderr:\n%s\n", out, err.Stderr) - } else { - log.Printf("Warning: Failed to run go list: %s", err.Error()) - } - return "", err - } - - return strings.TrimSpace(string(out)), nil -} - -// PkgInfo holds package directory and module directory (if any) for a package -type PkgInfo struct { - PkgDir string // the directory directly containing source code of this package - ModDir string // the module directory containing this package, empty if not a module -} - -// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the -// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps` -// is true, all dependencies will also be included. -func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]PkgInfo, error) { - // enable module mode so that we can find a module root if it exists, even if go module support is - // disabled by a build - if includingDeps { - // the flag `-deps` causes all dependencies to be retrieved - flags = append(flags, "-deps") - } - - // using -json overrides -f format - output, err := runGoList("", patterns, append(flags, "-json")...) - if err != nil { - return nil, err - } - - // the output of `go list -json` is a stream of json object - type goListPkgInfo struct { - ImportPath string - Dir string - Module *struct { - Dir string - } - } - pkgInfoMapping := make(map[string]PkgInfo) - streamDecoder := json.NewDecoder(strings.NewReader(output)) - for { - var pkgInfo goListPkgInfo - decErr := streamDecoder.Decode(&pkgInfo) - if decErr == io.EOF { - break - } - if decErr != nil { - log.Printf("Error decoding output of go list -json: %s", err.Error()) - return nil, decErr - } - pkgAbsDir, err := filepath.Abs(pkgInfo.Dir) - if err != nil { - log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error()) - } - var modAbsDir string - if pkgInfo.Module != nil { - modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir) - if err != nil { - log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error()) - } - } - pkgInfoMapping[pkgInfo.ImportPath] = PkgInfo{ - PkgDir: pkgAbsDir, - ModDir: modAbsDir, - } - } - return pkgInfoMapping, nil -} - -// GetPkgInfo fills the package info structure for the specified package path. -// It passes the `go list` the flags specified by `flags`. -func GetPkgInfo(pkgpath string, flags ...string) PkgInfo { - return PkgInfo{ - PkgDir: GetPkgDir(pkgpath, flags...), - ModDir: GetModDir(pkgpath, flags...), - } -} - -// GetModDir gets the absolute directory of the module containing the package with path -// `pkgpath`. It passes the `go list` the flags specified by `flags`. -func GetModDir(pkgpath string, flags ...string) string { - // enable module mode so that we can find a module root if it exists, even if go module support is - // disabled by a build - mod, err := runGoListWithEnv("{{.Module}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...) - if err != nil || mod == "" { - // if the command errors or modules aren't being used, return the empty string - return "" - } - - modDir, err := runGoListWithEnv("{{.Module.Dir}}", []string{pkgpath}, []string{"GO111MODULE=on"}, flags...) - if err != nil { - return "" - } - - abs, err := filepath.Abs(modDir) - if err != nil { - log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error()) - return "" - } - return abs -} - -// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the -// `go list` command the flags specified by `flags`. -func GetPkgDir(pkgpath string, flags ...string) string { - pkgDir, err := runGoList("{{.Dir}}", []string{pkgpath}, flags...) - if err != nil { - return "" - } - - abs, err := filepath.Abs(pkgDir) - if err != nil { - log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error()) - return "" - } - return abs -} - -// DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go -// list` command the flags specified by `flags`. -func DepErrors(pkgpath string, flags ...string) bool { - out, err := runGoList("{{if .DepsErrors}}error{{else}}{{end}}", []string{pkgpath}, flags...) - if err != nil { - // if go list failed, assume dependencies are broken - return false - } - - return out != "" -} - // FileExists tests whether the file at `filename` exists and is not a directory. func FileExists(filename string) bool { info, err := os.Stat(filename)