Skip to content

Commit e5a2b60

Browse files
author
Sauyon Lee
authored
Merge pull request #554 from xhd2015/accelerate_go_list
Accelerating go-extractor by using 'go list -deps' instead of just 'go list'
2 parents ee893b2 + f9ce06b commit e5a2b60

2 files changed

Lines changed: 96 additions & 60 deletions

File tree

‎extractor/extractor.go‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,19 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
102102
extractUniverseScope()
103103
log.Println("Done extracting universe scope.")
104104

105-
// a map of package path to package root directory (currently the module root or the source directory)
106-
pkgRoots := make(map[string]string)
107-
// a map of package path to source code directory
108-
pkgDirs := make(map[string]string)
105+
// a map of package path to source directory and module root directory
106+
pkgInfos := make(map[string]*util.PkgInfo)
109107
// root directories of packages that we want to extract
110108
wantedRoots := make(map[string]bool)
111109

110+
log.Printf("Running go list to resolve package and module directories.")
111+
// get all packages information
112+
pkgInfos, err = util.GetPkgsInfo(patterns, true, modFlags...)
113+
if err != nil {
114+
log.Fatalf("Error getting dependency package or module directories: %v.", err)
115+
}
116+
log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))
117+
112118
// recursively visit all packages in depth-first order;
113119
// on the way down, associate each package scope with its corresponding package,
114120
// and on the way up extract the package's scope
@@ -117,18 +123,6 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
117123
}, func(pkg *packages.Package) {
118124
log.Printf("Processing package %s.", pkg.PkgPath)
119125

120-
if _, ok := pkgRoots[pkg.PkgPath]; !ok {
121-
mdir := util.GetModDir(pkg.PkgPath, modFlags...)
122-
pdir := util.GetPkgDir(pkg.PkgPath, modFlags...)
123-
// GetModDir returns the empty string if the module directory cannot be determined, e.g. if the package
124-
// is not using modules. If this is the case, fall back to the package directory
125-
if mdir == "" {
126-
mdir = pdir
127-
}
128-
pkgRoots[pkg.PkgPath] = mdir
129-
pkgDirs[pkg.PkgPath] = pdir
130-
}
131-
132126
log.Printf("Extracting types for package %s.", pkg.PkgPath)
133127

134128
tw, err := trap.NewWriter(pkg.PkgPath, pkg)
@@ -153,11 +147,21 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
153147
})
154148

155149
for _, pkg := range pkgs {
156-
if pkgRoots[pkg.PkgPath] == "" {
150+
info := pkgInfos[pkg.PkgPath]
151+
if info == nil {
152+
var err error
153+
info, err = util.GetPkgInfo(pkg.PkgPath)
154+
if err != nil {
155+
log.Fatalf("Unable to get a source directory for input package %s: %s", pkg.PkgPath, err)
156+
}
157+
}
158+
if info == nil || info.PkgDir == "" {
157159
log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)
158160
}
159-
wantedRoots[pkgRoots[pkg.PkgPath]] = true
160-
wantedRoots[pkgDirs[pkg.PkgPath]] = true
161+
wantedRoots[info.PkgDir] = true
162+
if info.ModDir != "" {
163+
wantedRoots[info.ModDir] = true
164+
}
161165
}
162166

163167
log.Println("Done processing dependencies.")
@@ -175,16 +179,18 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
175179
return true
176180
}, func(pkg *packages.Package) {
177181
for root, _ := range wantedRoots {
178-
relDir, err := filepath.Rel(root, pkgDirs[pkg.PkgPath])
182+
info := pkgInfos[pkg.PkgPath]
183+
184+
relDir, err := filepath.Rel(root, info.PkgDir)
179185
if err != nil || noExtractRe.MatchString(relDir) {
180186
// if the path can't be made relative or matches the noExtract regexp skip it
181187
continue
182188
}
183189

184190
extraction.extractPackage(pkg)
185191

186-
if pkgRoots[pkg.PkgPath] != "" {
187-
modPath := filepath.Join(pkgRoots[pkg.PkgPath], "go.mod")
192+
if info.ModDir != "" {
193+
modPath := filepath.Join(info.ModDir, "go.mod")
188194
if util.FileExists(modPath) {
189195
log.Printf("Extracting %s", modPath)
190196
start := time.Now()

‎extractor/util/util.go‎

Lines changed: 68 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package util
22

33
import (
4+
"encoding/json"
45
"errors"
6+
"io"
57
"log"
68
"os"
79
"os/exec"
@@ -12,9 +14,9 @@ import (
1214

1315
var extractorPath string
1416

15-
// Getenv retrieves the value of the environment variable named by the key.
16-
// If that variable is not present, it iterates over the given aliases until
17-
// it finds one that is. If none are present, the empty string is returned.
17+
// Getenv retrieves the value of the environment variable named by the key. If that variable is not
18+
// present, it iterates over the given aliases until it finds one that is. If none are present, the
19+
// empty string is returned.
1820
func Getenv(key string, aliases ...string) string {
1921
val := os.Getenv(key)
2022
if val != "" {
@@ -30,16 +32,12 @@ func Getenv(key string, aliases ...string) string {
3032
}
3133

3234
// runGoList is a helper function for running go list with format `format` and flags `flags` on
33-
// package `pkgpath`.
34-
func runGoList(format string, pkgpath string, flags ...string) (string, error) {
35-
return runGoListWithEnv(format, pkgpath, nil, flags...)
36-
}
37-
38-
func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, flags ...string) (string, error) {
35+
// the packages defined by `patterns`.
36+
func runGoList(format string, patterns []string, flags ...string) (string, error) {
3937
args := append([]string{"list", "-e", "-f", format}, flags...)
40-
args = append(args, pkgpath)
38+
args = append(args, patterns...)
4139
cmd := exec.Command("go", args...)
42-
cmd.Env = append(os.Environ(), additionalEnv...)
40+
log.Printf("Running cmd: %s", cmd.String())
4341
out, err := cmd.Output()
4442

4543
if err != nil {
@@ -54,50 +52,82 @@ func runGoListWithEnv(format string, pkgpath string, additionalEnv []string, fla
5452
return strings.TrimSpace(string(out)), nil
5553
}
5654

57-
// GetModDir gets the absolute directory of the module containing the package with path
58-
// `pkgpath`. It passes the `go list` the flags specified by `flags`.
59-
func GetModDir(pkgpath string, flags ...string) string {
55+
// PkgInfo holds package directory and module directory (if any) for a package
56+
type PkgInfo struct {
57+
PkgDir string // the directory directly containing source code of this package
58+
ModDir string // the module directory containing this package, empty if not a module
59+
}
60+
61+
// GetPkgsInfo gets the absolute module and package root directories for the packages matched by the
62+
// patterns `patterns`. It passes to `go list` the flags specified by `flags`. If `includingDeps`
63+
// is true, all dependencies will also be included.
64+
func GetPkgsInfo(patterns []string, includingDeps bool, flags ...string) (map[string]*PkgInfo, error) {
6065
// enable module mode so that we can find a module root if it exists, even if go module support is
6166
// disabled by a build
62-
mod, err := runGoListWithEnv("{{.Module}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
63-
if err != nil || mod == "<nil>" {
64-
// if the command errors or modules aren't being used, return the empty string
65-
return ""
67+
if includingDeps {
68+
// the flag `-deps` causes all dependencies to be retrieved
69+
flags = append(flags, "-deps")
6670
}
6771

68-
modDir, err := runGoListWithEnv("{{.Module.Dir}}", pkgpath, []string{"GO111MODULE=on"}, flags...)
72+
// using -json overrides -f format
73+
output, err := runGoList("", patterns, append(flags, "-json")...)
6974
if err != nil {
70-
return ""
75+
return nil, err
7176
}
7277

73-
abs, err := filepath.Abs(modDir)
74-
if err != nil {
75-
log.Printf("Warning: unable to make %s absolute: %s", modDir, err.Error())
76-
return ""
78+
// the output of `go list -json` is a stream of json object
79+
type goListPkgInfo struct {
80+
ImportPath string
81+
Dir string
82+
Module *struct {
83+
Dir string
84+
}
7785
}
78-
return abs
79-
}
80-
81-
// GetPkgDir gets the absolute directory containing the package with path `pkgpath`. It passes the
82-
// `go list` command the flags specified by `flags`.
83-
func GetPkgDir(pkgpath string, flags ...string) string {
84-
pkgDir, err := runGoList("{{.Dir}}", pkgpath, flags...)
85-
if err != nil {
86-
return ""
86+
pkgInfoMapping := make(map[string]*PkgInfo)
87+
streamDecoder := json.NewDecoder(strings.NewReader(output))
88+
for {
89+
var pkgInfo goListPkgInfo
90+
decErr := streamDecoder.Decode(&pkgInfo)
91+
if decErr == io.EOF {
92+
break
93+
}
94+
if decErr != nil {
95+
log.Printf("Error decoding output of go list -json: %s", err.Error())
96+
return nil, decErr
97+
}
98+
pkgAbsDir, err := filepath.Abs(pkgInfo.Dir)
99+
if err != nil {
100+
log.Printf("Unable to make package dir %s absolute: %s", pkgInfo.Dir, err.Error())
101+
}
102+
var modAbsDir string
103+
if pkgInfo.Module != nil {
104+
modAbsDir, err = filepath.Abs(pkgInfo.Module.Dir)
105+
if err != nil {
106+
log.Printf("Unable to make module dir %s absolute: %s", pkgInfo.Module.Dir, err.Error())
107+
}
108+
}
109+
pkgInfoMapping[pkgInfo.ImportPath] = &PkgInfo{
110+
PkgDir: pkgAbsDir,
111+
ModDir: modAbsDir,
112+
}
87113
}
114+
return pkgInfoMapping, nil
115+
}
88116

89-
abs, err := filepath.Abs(pkgDir)
117+
// GetPkgsInfo gets the absolute module and package root directories for the package `pkg`, passing
118+
// the internal `go list` command the flags specified by `flags`.
119+
func GetPkgInfo(pkg string, flags ...string) (*PkgInfo, error) {
120+
pkgInfos, err := GetPkgsInfo([]string{pkg}, false, flags...)
90121
if err != nil {
91-
log.Printf("Warning: unable to make %s absolute: %s", pkgDir, err.Error())
92-
return ""
122+
return nil, err
93123
}
94-
return abs
124+
return pkgInfos[pkg], nil
95125
}
96126

97127
// DepErrors checks there are any errors resolving dependencies for `pkgpath`. It passes the `go
98128
// list` command the flags specified by `flags`.
99129
func DepErrors(pkgpath string, flags ...string) bool {
100-
out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", pkgpath, flags...)
130+
out, err := runGoList("{{if .DepsErrors}}{{else}}error{{end}}", []string{pkgpath}, flags...)
101131
if err != nil {
102132
// if go list failed, assume dependencies are broken
103133
return false

0 commit comments

Comments
 (0)