Rewrite update to consider new transitive dependencies

.vscode/settings.json

@@ -0,0 +1,3 @@
+// Place your settings in this file to overwrite default and user settings.
+{
+}

.vscode/tasks.json

@@ -0,0 +1,16 @@
+{
+	// See http://go.microsoft.com/fwlink/?LinkId=733558
+	// for the documentation about the tasks.json format
+	"version": "0.1.0",
+	"command": "go",
+	"isShellCommand": true,
+	"args": ["test"],
+	"showOutput": "always",
+	"tasks": [
+		{
+			"taskName":"Local Test",
+			"args":[ "-v", "-run=Update"],
+			"suppressTaskName": true
+		}
+	]
+}

Changelog.md

@@ -1,3 +1,7 @@
+#v65 (2016/06/09)
+
+* Rewrite update so that it considers new transitive dependencies, both in the same repo and outside of it.
+
 #v64 (2016/06/09)
 
 * godep update golang.org/x/tools/go/vcs

godepfile.go

@@ -185,3 +185,23 @@ func (g *Godeps) writeTo(w io.Writer) (int64, error) {
 	n, err := w.Write(append(b, '\n'))
 	return int64(n), err
 }
+
+func (g *Godeps) addOrUpdateDeps(deps []Dependency) {
+	var missing []Dependency
+	for _, d := range deps {
+		var found bool
+		for i := range g.Deps {
+			if g.Deps[i].ImportPath == d.ImportPath {
+				g.Deps[i] = d
+				found = true
+				break
+			}
+		}
+		if !found {
+			missing = append(missing, d)
+		}
+	}
+	for _, d := range missing {
+		g.Deps = append(g.Deps, d)
+	}
+}

list.go

@@ -164,6 +164,7 @@ func listPackage(path string) (*Package, error) {
 			p.Imports = append(p.Imports, dp.ImportPath)
 		}
 		p.Deps = append(p.Deps, dp.ImportPath)
+		p.Dependencies = addDependency(p.Dependencies, dp)
 	}
 	p.Imports = uniq(p.Imports)
 	p.Deps = uniq(p.Deps)
@@ -172,6 +173,15 @@ func listPackage(path string) (*Package, error) {
 	return p, nil
 }
 
+func addDependency(deps []build.Package, d *build.Package) []build.Package {
+	for i := range deps {
+		if deps[i].Dir == d.Dir {
+			return deps
+		}
+	}
+	return append(deps, *d)
+}
+
 // finds the directory for the given import path in the context of the provided build.Package (if provided)
 func findDirForPath(path string, ip *build.Package) (string, error) {
 	debugln("findDirForPath", path, ip)
@@ -206,7 +216,7 @@ func findDirForPath(path string, ip *build.Package) (string, error) {
 
 	for _, dir := range search {
 		debugln("searching", dir)
-		fi, err := os.Stat(dir)
+		fi, err := stat(dir)
 		if err == nil && fi.IsDir() {
 			return dir, nil
 		}
@@ -215,6 +225,28 @@ func findDirForPath(path string, ip *build.Package) (string, error) {
 	return "", errPackageNotFound{path}
 }
 
+type statEntry struct {
+	fi  os.FileInfo
+	err error
+}
+
+var (
+	statCache = make(map[string]statEntry)
+)
+
+func clearStatCache() {
+	statCache = make(map[string]statEntry)
+}
+
+func stat(p string) (os.FileInfo, error) {
+	if e, ok := statCache[p]; ok {
+		return e.fi, e.err
+	}
+	fi, err := os.Stat(p)
+	statCache[p] = statEntry{fi, err}
+	return fi, err
+}
+
 // fillPackage full of info. Assumes p.Dir is set at a minimum
 func fillPackage(p *build.Package) error {
 	if p.Goroot {

pkg.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"go/build"
 	"regexp"
 	"strings"
 )
@@ -28,7 +29,8 @@ type Package struct {
 	}
 
 	// --- New stuff for now
-	Imports []string
+	Imports      []string
+	Dependencies []build.Package
 }
 
 // LoadPackages loads the named packages using go list -json.

restore.go

@@ -83,7 +83,7 @@ func runRestore(cmd *Command, args []string) {
 
 var downloaded = make(map[string]bool)
 
-// download downloads the given dependency.
+// download the given dependency.
 // 2 Passes: 1) go get -d <pkg>, 2) git pull (if necessary)
 func download(dep *Dependency) error {
 

save.go

@@ -365,6 +365,9 @@ func copySrc(dir string, deps []Dependency) error {
 		debugln("copySrc for", dep.ImportPath)
 		srcdir := filepath.Join(dep.ws, "src")
 		rel, err := filepath.Rel(srcdir, dep.dir)
+		debugln("srcdir", srcdir)
+		debugln("rel", rel)
+		debugln("err", err)
 		if err != nil { // this should never happen
 			return err
 		}

save_test.go

@@ -89,6 +89,7 @@ func godeps(importpath string, keyval ...string) *Godeps {
 
 func setGlobals(vendor bool) {
 	clearPkgCache()
+	clearStatCache()
 	VendorExperiment = vendor
 	sep = defaultSep(VendorExperiment)
 	//debug = testing.Verbose()

update.go

@@ -101,6 +101,7 @@ func update(args []string) error {
 	if len(deps) == 0 {
 		return errorNoPackagesUpdatable
 	}
+	g.addOrUpdateDeps(deps)
 	if _, err = g.save(); err != nil {
 		return err
 	}
@@ -172,81 +173,107 @@ func markMatches(pat string, deps []Dependency) (matched bool) {
 	return matched
 }
 
+func fillDeps(deps []Dependency) ([]Dependency, error) {
+	for i := range deps {
+		if deps[i].pkg != nil {
+			continue
+		}
+		ps, err := LoadPackages(deps[i].ImportPath)
+		if err != nil {
+			return nil, err
+		}
+		if len(ps) > 1 {
+			panic("More than one package found for " + deps[i].ImportPath)
+		}
+		p := ps[0]
+		deps[i].pkg = p
+		deps[i].dir = p.Dir
+		deps[i].ws = p.Root
+
+		vcs, reporoot, err := VCSFromDir(p.Dir, filepath.Join(p.Root, "src"))
+		if err != nil {
+			return nil, errorLoadingDeps
+		}
+		deps[i].root = filepath.ToSlash(reporoot)
+		deps[i].vcs = vcs
+	}
+
+	return deps, nil
+}
+
 // LoadVCSAndUpdate loads and updates a set of dependencies.
 func LoadVCSAndUpdate(deps []Dependency) ([]Dependency, error) {
 	var err1 error
-	var paths []string
-	for _, dep := range deps {
-		paths = append(paths, dep.ImportPath)
-	}
-	ps, err := LoadPackages(paths...)
+
+	deps, err := fillDeps(deps)
 	if err != nil {
 		return nil, err
 	}
-	noupdate := make(map[string]bool) // repo roots
-	var candidates []*Dependency
-	var tocopy []Dependency
+
+	repoMask := make(map[string]bool)
 	for i := range deps {
-		dep := &deps[i]
-		for _, pkg := range ps {
-			if dep.ImportPath == pkg.ImportPath {
-				dep.pkg = pkg
-				break
-			}
-		}
-		if dep.pkg == nil {
-			log.Println(dep.ImportPath + ": error listing package")
-			err1 = errorLoadingDeps
-			continue
+		if !deps[i].matched {
+			repoMask[deps[i].root] = true
 		}
-		if dep.pkg.Error.Err != "" {
-			log.Println(dep.pkg.Error.Err)
-			err1 = errorLoadingDeps
-			continue
-		}
-		vcs, reporoot, err := VCSFromDir(dep.pkg.Dir, filepath.Join(dep.pkg.Root, "src"))
-		if err != nil {
-			log.Println(err)
-			err1 = errorLoadingDeps
+	}
+
+	// Determine if we need any new packages because of new transitive imports
+	for _, dep := range deps {
+		if !dep.matched {
 			continue
 		}
-		dep.dir = dep.pkg.Dir
-		dep.ws = dep.pkg.Root
-		dep.root = filepath.ToSlash(reporoot)
-		dep.vcs = vcs
-		if dep.matched {
-			candidates = append(candidates, dep)
-		} else {
-			noupdate[dep.root] = true
+		for _, dp := range dep.pkg.Dependencies {
+			if dp.Goroot {
+				continue
+			}
+			var have bool
+			for _, d := range deps {
+				if d.ImportPath == dp.ImportPath {
+					have = true
+					break
+				}
+			}
+			if !have {
+				deps = append(deps, Dependency{ImportPath: dp.ImportPath, matched: true})
+			}
 		}
 	}
-	if err1 != nil {
-		return nil, err1
+
+	deps, err = fillDeps(deps)
+	if err != nil {
+		return nil, err
 	}
 
-	for _, dep := range candidates {
-		dep.dir = dep.pkg.Dir
-		dep.ws = dep.pkg.Root
-		if noupdate[dep.root] {
+	var toUpdate []Dependency
+	for _, d := range deps {
+		if !d.matched || repoMask[d.root] {
 			continue
 		}
-		id, err := dep.vcs.identify(dep.pkg.Dir)
+		toUpdate = append(toUpdate, d)
+	}
+	debugln("toUpdate")
+	ppln(toUpdate)
+
+	var toCopy []Dependency
+	for _, d := range toUpdate {
+		id, err := d.vcs.identify(d.dir)
 		if err != nil {
 			log.Println(err)
 			err1 = errorLoadingDeps
 			continue
 		}
-		if dep.vcs.isDirty(dep.pkg.Dir, id) {
-			log.Println("dirty working tree (please commit changes):", dep.pkg.Dir)
-			err1 = errorLoadingDeps
-			break
+		if d.vcs.isDirty(d.dir, id) {
+			log.Println("dirty working tree (please commit changes):", d.dir)
 		}
-		dep.Rev = id
-		dep.Comment = dep.vcs.describe(dep.pkg.Dir, id)
-		tocopy = append(tocopy, *dep)
+		d.Rev = id
+		d.Comment = d.vcs.describe(d.dir, id)
+		toCopy = append(toCopy, d)
 	}
+	debugln("toCopy")
+	ppln(toCopy)
+
 	if err1 != nil {
 		return nil, err1
 	}
-	return tocopy, nil
+	return toCopy, nil
 }