cmd/go/internal/modfetch: move to new pseudo-version design

The original pseudo-version design used versions of the form

	v0.0.0-yyyymmddhhmmss-abcdef123456

These were intentionally chosen to be valid semantic versions
that sort below any explicitly-chosen semantic version (even v0.0.0),
so that they could be used before anything was tagged but after
that would essentially only be useful in replace statements
(because the max operation during MVS would always prefer
a tagged version).

Then we changed the go command to accept hashes on the
command line, so that you can say

	go get github.com/my/proj@abcdef

and it will download and use v0.0.0-yyyymmddhhmmss-abcdef123456.

If you were using v1.10.1 before and this commit is just little bit
newer than that commit, calling it v0.0.0-xxx is confusing but
also harmful: the go command sees the change from v1.10.1 to
the v0.0.0 pseudoversion as a downgrade, and it downgrades other
modules in the build. In particular if some other module has
a requirement of github.com/my/proj v1.9.0 (or later), the
pseudo-version appears to be before that, so go get would
downgrade that module too. It might even remove it entirely,
if every available version needs a post-v0.0.0 version of my/proj.

This CL introduces new pseudo-version forms that can be used
to slot in after the most recent explicit tag before the commit.
If the most recent tagged commit before abcdef is v1.10.1,
then now we will use

	v1.10.2-0.yyyymmddhhmmss-abcdef123456

This has the right properties for downgrades and the like,
since it is after v1.10.1 but before almost any possible
successor, such as v1.10.2, v1.10.2-1, or v1.10.2-pre.

This CL also uses those pseudo-version forms as appropriate
when mapping a hash to a pseudo-version. This fixes the
downgrade problem.

Overall, this CL reflects our growing recognition of pseudo-versions
as being like "untagged prereleases".

Issue #26150 was about documenting best practices for how
to work around this kind of accidental downgrade problem
with additional steps. Now there are no additional steps:
the problem is avoided by default.

Fixes #26150.

Change-Id: I402feeccb93e8e937bafcaa26402d88572e9b14c
Reviewed-on: https://go-review.googlesource.com/124515
Reviewed-by: Bryan C. Mills <[email protected]>

Author: rsc

src/cmd/go/internal/modfetch/codehost/codehost.go

@@ -77,6 +77,15 @@ type Repo interface {
 	// contained in the zip file. All files in the zip file are expected to be
 	// nested in a single top-level directory, whose name is not specified.
 	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error)
+
+	// RecentTag returns the most recent tag at or before the given rev
+	// with the given prefix. It should make a best-effort attempt to
+	// find a tag that is a valid semantic version (following the prefix),
+	// or else the result is not useful to the caller, but it need not
+	// incur great expense in doing so. For example, the git implementation
+	// of RecentTag limits git's search to tags matching the glob expression
+	// "v[0-9]*.[0-9]*.[0-9]*" (after the prefix).
+	RecentTag(rev, prefix string) (tag string, err error)
 }
 
 // A Rev describes a single revision in a source code repository.

src/cmd/go/internal/modfetch/codehost/git.go

@@ -602,6 +602,18 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F
 	return missing, nil
 }
 
+func (r *gitRepo) RecentTag(rev, prefix string) (tag string, err error) {
+	_, err = r.Stat(rev)
+	if err != nil {
+		return "", err
+	}
+	out, err := Run(r.dir, "git", "describe", "--first-parent", "--tags", "--always", "--abbrev=0", "--match", prefix+"v[0-9]*.[0-9]*.[0-9]*", "--tags", rev)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
 func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
 	// TODO: Use maxSize or drop it.
 	args := []string{}

src/cmd/go/internal/modfetch/codehost/vcs.go

@@ -329,6 +329,10 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s
 	return nil, fmt.Errorf("ReadFileRevs not implemented")
 }
 
+func (r *vcsRepo) RecentTag(rev, prefix string) (tag string, err error) {
+	return "", fmt.Errorf("RecentTags not implemented")
+}
+
 func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
 	if rev == "latest" {
 		rev = r.cmd.latest

src/cmd/go/internal/modfetch/coderepo.go

@@ -6,15 +6,12 @@ package modfetch
 
 import (
 	"archive/zip"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
-	"regexp"
 	"strings"
-	"time"
 
 	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/modfile"
@@ -194,7 +191,7 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 			}
 		}
 
-		tagOK := func(v string) string {
+		tagToVersion := func(v string) string {
 			if !strings.HasPrefix(v, p) {
 				return ""
 			}
@@ -212,26 +209,28 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 		}
 
 		// If info.Version is OK, use it.
-		if v := tagOK(info.Version); v != "" {
+		if v := tagToVersion(info.Version); v != "" {
 			info2.Version = v
 		} else {
 			// Otherwise look through all known tags for latest in semver ordering.
 			for _, tag := range info.Tags {
-				if v := tagOK(tag); v != "" && semver.Compare(info2.Version, v) < 0 {
+				if v := tagToVersion(tag); v != "" && semver.Compare(info2.Version, v) < 0 {
 					info2.Version = v
 				}
 			}
 			// Otherwise make a pseudo-version.
 			if info2.Version == "" {
-				info2.Version = PseudoVersion(r.pseudoMajor, info.Time, info.Short)
+				tag, _ := r.code.RecentTag(statVers, p)
+				v = tagToVersion(tag)
+				// TODO: Check that v is OK for r.pseudoMajor or else is OK for incompatible.
+				info2.Version = PseudoVersion(r.pseudoMajor, v, info.Time, info.Short)
 			}
 		}
 	}
 
 	// Do not allow a successful stat of a pseudo-version for a subdirectory
 	// unless the subdirectory actually does have a go.mod.
 	if IsPseudoVersion(info2.Version) && r.codeDir != "" {
-		// TODO: git describe --first-parent --match 'v[0-9]*' --tags
 		_, _, _, err := r.findDir(info2.Version)
 		if err != nil {
 			// TODO: It would be nice to return an error like "not a module".
@@ -246,9 +245,8 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 func (r *codeRepo) revToRev(rev string) string {
 	if semver.IsValid(rev) {
 		if IsPseudoVersion(rev) {
-			i := strings.Index(rev, "-")
-			j := strings.Index(rev[i+1:], "-")
-			return rev[i+1+j+1:]
+			r, _ := PseudoVersionRev(rev)
+			return r
 		}
 		if semver.Build(rev) == "+incompatible" {
 			rev = rev[:len(rev)-len("+incompatible")]
@@ -598,71 +596,3 @@ func isVendoredPackage(name string) bool {
 	}
 	return strings.Contains(name[i:], "/")
 }
-
-func PseudoVersion(major string, t time.Time, rev string) string {
-	if major == "" {
-		major = "v0"
-	}
-	return fmt.Sprintf("%s.0.0-%s-%s", major, t.UTC().Format("20060102150405"), rev)
-}
-
-var ErrNotPseudoVersion = errors.New("not a pseudo-version")
-
-/*
-func ParsePseudoVersion(repo Repo, version string) (rev string, err error) {
-	major := semver.Major(version)
-	if major == "" {
-		return "", ErrNotPseudoVersion
-	}
-	majorPrefix := major + ".0.0-"
-	if !strings.HasPrefix(version, majorPrefix) || !strings.Contains(version[len(majorPrefix):], "-") {
-		return "", ErrNotPseudoVersion
-	}
-	versionSuffix := version[len(majorPrefix):]
-	for i := 0; versionSuffix[i] != '-'; i++ {
-		c := versionSuffix[i]
-		if c < '0' || '9' < c {
-			return "", ErrNotPseudoVersion
-		}
-	}
-	rev = versionSuffix[strings.Index(versionSuffix, "-")+1:]
-	if rev == "" {
-		return "", ErrNotPseudoVersion
-	}
-	if proxyURL != "" {
-		return version, nil
-	}
-	fullRev, t, err := repo.CommitInfo(rev)
-	if err != nil {
-		return "", fmt.Errorf("unknown pseudo-version %s: loading %v: %v", version, rev, err)
-	}
-	v := PseudoVersion(major, t, repo.ShortRev(fullRev))
-	if v != version {
-		return "", fmt.Errorf("unknown pseudo-version %s: %v is %v", version, rev, v)
-	}
-	return fullRev, nil
-}
-*/
-
-var pseudoVersionRE = regexp.MustCompile(`^v[0-9]+\.0\.0-[0-9]{14}-[A-Za-z0-9]+$`)
-
-// IsPseudoVersion reports whether v is a pseudo-version.
-func IsPseudoVersion(v string) bool {
-	return pseudoVersionRE.MatchString(v)
-}
-
-// PseudoVersionTime returns the time stamp of the pseudo-version v.
-// It returns an error if v is not a pseudo-version or if the time stamp
-// embedded in the pseudo-version is not a valid time.
-func PseudoVersionTime(v string) (time.Time, error) {
-	if !IsPseudoVersion(v) {
-		return time.Time{}, fmt.Errorf("not a pseudo-version")
-	}
-	i := strings.Index(v, "-") + 1
-	j := i + strings.Index(v[i:], "-")
-	t, err := time.Parse("20060102150405", v[i:j])
-	if err != nil {
-		return time.Time{}, fmt.Errorf("malformed pseudo-version %q", v)
-	}
-	return t, nil
-}

src/cmd/go/internal/modfetch/coderepo_test.go

@@ -237,7 +237,7 @@ var codeRepoTests = []struct {
 		// redirect to googlesource
 		path:    "golang.org/x/text",
 		rev:     "4e4a3210bb",
-		version: "v0.0.0-20180208041248-4e4a3210bb54",
+		version: "v0.3.1-0.20180208041248-4e4a3210bb54",
 		name:    "4e4a3210bb54bb31f6ab2cdca2edcc0b50c420c1",
 		short:   "4e4a3210bb54",
 		time:    time.Date(2018, 2, 8, 4, 12, 48, 0, time.UTC),
@@ -611,6 +611,9 @@ func (ch *fixedTagsRepo) ReadFileRevs([]string, string, int64) (map[string]*code
 func (ch *fixedTagsRepo) ReadZip(string, string, int64) (io.ReadCloser, string, error) {
 	panic("not impl")
 }
+func (ch *fixedTagsRepo) RecentTag(string, string) (string, error) {
+	panic("not impl")
+}
 func (ch *fixedTagsRepo) Stat(string) (*codehost.RevInfo, error) { panic("not impl") }
 
 func TestNonCanonicalSemver(t *testing.T) {

src/cmd/go/internal/modfetch/pseudo.go

@@ -0,0 +1,128 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Pseudo-versions
+//
+// Code authors are expected to tag the revisions they want users to use,
+// including prereleases. However, not all authors tag versions at all,
+// and not all commits a user might want to try will have tags.
+// A pseudo-version is a version with a special form that allows us to
+// address an untagged commit and order that version with respect to
+// other versions we might encounter.
+//
+// A pseudo-version takes one of the general forms:
+//
+//	(1) vX.0.0-yyyymmddhhmmss-abcdef123456
+//	(2) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456
+//	(3) vX.Y.(Z+1)-0.yyyymmddhhmmss-abcdef123456+incompatible
+//	(4) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456
+//	(5) vX.Y.Z-pre.0.yyyymmddhhmmss-abcdef123456+incompatible
+//
+// If there is no recently tagged version with the right major version vX,
+// then form (1) is used, creating a space of pseudo-versions at the bottom
+// of the vX version range, less than any tagged version, including the unlikely v0.0.0.
+//
+// If the most recent tagged version before the target commit is vX.Y.Z or vX.Y.Z+incompatible,
+// then the pseudo-version uses form (2) or (3), making it a prerelease for the next
+// possible semantic version after vX.Y.Z. The leading 0 segment in the prerelease string
+// ensures that the pseudo-version compares less than possible future explicit prereleases
+// like vX.Y.(Z+1)-rc1 or vX.Y.(Z+1)-1.
+//
+// If the most recent tagged version before the target commit is vX.Y.Z-pre or vX.Y.Z-pre+incompatible,
+// then the pseudo-version uses form (4) or (5), making it a slightly later prerelease.
+
+package modfetch
+
+import (
+	"cmd/go/internal/semver"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+)
+
+// PseudoVersion returns a pseudo-version for the given major version ("v1")
+// preexisting older tagged version ("" or "v1.2.3" or "v1.2.3-pre"), revision time,
+// and revision identifier (usually a 12-byte commit hash prefix).
+func PseudoVersion(major, older string, t time.Time, rev string) string {
+	if major == "" {
+		major = "v0"
+	}
+	segment := fmt.Sprintf("%s-%s", t.UTC().Format("20060102150405"), rev)
+	build := semver.Build(older)
+	older = semver.Canonical(older)
+	if older == "" {
+		return major + ".0.0-" + segment // form (1)
+	}
+	if semver.Prerelease(older) != "" {
+		return older + ".0." + segment + build // form (4), (5)
+	}
+
+	// Form (2), (3).
+	// Extract patch from vMAJOR.MINOR.PATCH
+	v := older[:len(older)]
+	i := strings.LastIndex(v, ".") + 1
+	v, patch := v[:i], v[i:]
+
+	// Increment PATCH by adding 1 to decimal:
+	// scan right to left turning 9s to 0s until you find a digit to increment.
+	// (Number might exceed int64, but math/big is overkill.)
+	digits := []byte(patch)
+	for i = len(digits) - 1; i >= 0 && digits[i] == '9'; i-- {
+		digits[i] = '0'
+	}
+	if i >= 0 {
+		digits[i]++
+	} else {
+		// digits is all zeros
+		digits[0] = '1'
+		digits = append(digits, '0')
+	}
+	patch = string(digits)
+
+	// Reassemble.
+	return v + patch + "-0." + segment + build
+}
+
+var pseudoVersionRE = regexp.MustCompile(`^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)\d{14}-[A-Za-z0-9]+(\+incompatible)?$`)
+
+// IsPseudoVersion reports whether v is a pseudo-version.
+func IsPseudoVersion(v string) bool {
+	return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v)
+}
+
+// PseudoVersionTime returns the time stamp of the pseudo-version v.
+// It returns an error if v is not a pseudo-version or if the time stamp
+// embedded in the pseudo-version is not a valid time.
+func PseudoVersionTime(v string) (time.Time, error) {
+	timestamp, _, err := parsePseudoVersion(v)
+	t, err := time.Parse("20060102150405", timestamp)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("pseudo-version with malformed time %s: %q", timestamp, v)
+	}
+	return t, nil
+}
+
+// PseudoVersionRev returns the revision identifier of the pseudo-version v.
+// It returns an error if v is not a pseudo-version.
+func PseudoVersionRev(v string) (rev string, err error) {
+	_, rev, err = parsePseudoVersion(v)
+	return
+}
+
+func parsePseudoVersion(v string) (timestamp, rev string, err error) {
+	if !IsPseudoVersion(v) {
+		return "", "", fmt.Errorf("malformed pseudo-version %q", v)
+	}
+	v = strings.TrimSuffix(v, "+incompatible")
+	j := strings.LastIndex(v, "-")
+	v, rev = v[:j], v[j+1:]
+	i := strings.LastIndex(v, "-")
+	if j := strings.LastIndex(v, "."); j > i {
+		timestamp = v[j+1:]
+	} else {
+		timestamp = v[i+1:]
+	}
+	return timestamp, rev, nil
+}