fix

Author: wxiaoguang

modules/git/batch_reader.go

@@ -242,15 +242,15 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
 	return out
 }
 
-// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
+// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream
 // This carefully avoids allocations - except where fnameBuf is too small.
 // It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
 //
 // Each line is composed of:
 // <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <binary HASH>
 //
 // We don't attempt to convert the raw HASH to save a lot of time
-func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
+func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
 	var readBytes []byte
 
 	// Read the Mode & fname
@@ -260,7 +260,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu
 	}
 	idx := bytes.IndexByte(readBytes, ' ')
 	if idx < 0 {
-		log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes)
+		log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes)
 		return mode, fname, sha, n, &ErrNotExist{}
 	}
 

modules/git/commit_submodule_file.go

@@ -5,17 +5,12 @@
 package git
 
 import (
-	"bufio"
-	"context"
 	"fmt"
-	"io"
 	"net"
 	"net/url"
 	"path"
 	"regexp"
 	"strings"
-
-	"code.gitea.io/gitea/modules/log"
 )
 
 var scpSyntax = regexp.MustCompile(`^([a-zA-Z0-9_]+@)?([a-zA-Z0-9._-]+):(.*)$`)
@@ -114,101 +109,3 @@ func (sf *CommitSubModuleFile) RefURL(urlPrefix, repoFullName, sshDomain string)
 func (sf *CommitSubModuleFile) RefID() string {
 	return sf.refID
 }
-
-// SubModuleCommit submodule name and commit from a repository
-type SubModuleCommit struct {
-	Name   string
-	Commit string
-}
-
-// GetSubmoduleCommits Returns a list of active submodules in the repository
-func GetSubmoduleCommits(ctx context.Context, repoPath string) []SubModuleCommit {
-	stdoutReader, stdoutWriter := io.Pipe()
-	defer func() {
-		_ = stdoutReader.Close()
-		_ = stdoutWriter.Close()
-	}()
-
-	go func() {
-		stderrBuilder := strings.Builder{}
-		err := NewCommand(ctx, "config", "-f", ".gitmodules", "--list", "--name-only").Run(&RunOpts{
-			Timeout: -1,
-			Dir:     repoPath,
-			Stdout:  stdoutWriter,
-			Stderr:  &stderrBuilder,
-		})
-
-		if err != nil {
-			_ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String()))
-		} else {
-			_ = stdoutWriter.Close()
-		}
-	}()
-
-	var submodules []SubModuleCommit
-	bufReader := bufio.NewReader(stdoutReader)
-
-	for {
-		line, err := bufReader.ReadString('\n')
-		if err != nil {
-			break
-		}
-
-		if len(line) < len("submodule.x.url\n") ||
-			!strings.HasPrefix(line, "submodule.") ||
-			!strings.HasSuffix(line, ".url\n") {
-
-			continue
-		}
-
-		name := line[len("submodule.") : len(line)-len(".url\n")]
-		name = strings.TrimSpace(name)
-
-		if len(name) == 0 {
-			log.Debug("Submodule skipped because it has no name")
-			continue
-		}
-
-		// If no commit was found for the module skip it
-		commit, _, err := NewCommand(ctx, "submodule", "status").AddDynamicArguments(name).RunStdString(&RunOpts{Dir: repoPath})
-		if err != nil {
-			log.Debug("Submodule %s skipped because it has no commit", name)
-			continue
-		}
-
-		if len(commit) > 0 {
-			commit = commit[1:]
-		}
-
-		fields := strings.Fields(commit)
-
-		if len(fields) == 0 {
-			log.Debug("Submodule %s skipped because it has no valid commit", name)
-			continue
-		}
-
-		commit = fields[0]
-
-		if len(commit) != 40 {
-			log.Debug("Submodule %s skipped due to malformed commit hash", name)
-			continue
-		}
-
-		submodules = append(submodules, SubModuleCommit{name, commit})
-	}
-
-	return submodules
-}
-
-// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present.
-func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error {
-	for _, submodule := range submodules {
-		if stdout, _, err := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Name).
-			RunStdString(&RunOpts{Dir: repoPath}); err != nil {
-			log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Name, repoPath, stdout, err)
-			return err
-		}
-	}
-
-	return nil
-}

modules/git/commit_submodule_file_test.go

@@ -4,10 +4,8 @@
 package git
 
 import (
-	"path/filepath"
 	"testing"
 
-	"code.gitea.io/gitea/modules/util"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -42,20 +40,3 @@ func TestCommitSubModuleFileGetRefURL(t *testing.T) {
 		assert.EqualValues(t, kase.expect, getRefURL(kase.refURL, kase.prefixURL, kase.parentPath, kase.SSHDomain))
 	}
 }
-
-func TestRepository_GetSubmoduleCommits(t *testing.T) {
-	bareRepo1Path := filepath.Join(testReposDir, "repo4_submodules")
-	clonedPath, err := cloneRepo(bareRepo1Path, "repo4_TestRepository_GetSubmoduleCommits")
-	assert.NoError(t, err)
-	defer util.RemoveAll(clonedPath)
-
-	submodules := GetSubmoduleCommits(DefaultContext, clonedPath)
-
-	assert.EqualValues(t, len(submodules), 2)
-
-	assert.EqualValues(t, submodules[0].Name, "libtest")
-	assert.EqualValues(t, submodules[0].Commit, "1234567890123456789012345678901234567890")
-
-	assert.EqualValues(t, submodules[1].Name, "<°)))><")
-	assert.EqualValues(t, submodules[1].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c")
-}

modules/git/parse_nogogit.go

@@ -23,68 +23,79 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
 
 var sepSpace = []byte{' '}
 
-func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
+func parseLsTreeLine(line []byte) (*TreeEntry, error) {
+	// expect line to be of the form:
+	// <mode> <type> <sha> <space-padded-size>\t<filename>
+	// <mode> <type> <sha>\t<filename>
+
 	var err error
+	posTab := bytes.IndexByte(line, '\t')
+	if posTab == -1 {
+		return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
+	}
+
+	entry := new(TreeEntry)
+
+	entryAttrs := line[:posTab]
+	entryName := line[posTab+1:]
+
+	entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
+	_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
+	entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
+	if len(entryAttrs) > 0 {
+		entrySize := entryAttrs // the last field is the space-padded-size
+		entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
+		entry.sized = true
+	}
+
+	switch string(entryMode) {
+	case "100644":
+		entry.entryMode = EntryModeBlob
+	case "100755":
+		entry.entryMode = EntryModeExec
+	case "120000":
+		entry.entryMode = EntryModeSymlink
+	case "160000":
+		entry.entryMode = EntryModeCommit
+	case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
+		entry.entryMode = EntryModeTree
+	default:
+		return nil, fmt.Errorf("unknown type: %v", string(entryMode))
+	}
+
+	entry.ID, err = NewIDFromString(string(entryObjectID))
+	if err != nil {
+		return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
+	}
+
+	if len(entryName) > 0 && entryName[0] == '"' {
+		entry.name, err = strconv.Unquote(string(entryName))
+		if err != nil {
+			return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
+		}
+	} else {
+		entry.name = string(entryName)
+	}
+	return entry, nil
+}
+
+// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory
+func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
 	entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
 	for pos := 0; pos < len(data); {
-		// expect line to be of the form:
-		// <mode> <type> <sha> <space-padded-size>\t<filename>
-		// <mode> <type> <sha>\t<filename>
 		posEnd := bytes.IndexByte(data[pos:], '\n')
 		if posEnd == -1 {
 			posEnd = len(data)
 		} else {
 			posEnd += pos
 		}
-		line := data[pos:posEnd]
-		posTab := bytes.IndexByte(line, '\t')
-		if posTab == -1 {
-			return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
-		}
-
-		entry := new(TreeEntry)
-		entry.ptree = ptree
 
-		entryAttrs := line[:posTab]
-		entryName := line[posTab+1:]
-
-		entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
-		_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
-		entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
-		if len(entryAttrs) > 0 {
-			entrySize := entryAttrs // the last field is the space-padded-size
-			entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
-			entry.sized = true
-		}
-
-		switch string(entryMode) {
-		case "100644":
-			entry.entryMode = EntryModeBlob
-		case "100755":
-			entry.entryMode = EntryModeExec
-		case "120000":
-			entry.entryMode = EntryModeSymlink
-		case "160000":
-			entry.entryMode = EntryModeCommit
-		case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
-			entry.entryMode = EntryModeTree
-		default:
-			return nil, fmt.Errorf("unknown type: %v", string(entryMode))
-		}
-
-		entry.ID, err = NewIDFromString(string(entryObjectID))
+		line := data[pos:posEnd]
+		entry, err := parseLsTreeLine(line)
 		if err != nil {
-			return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
-		}
-
-		if len(entryName) > 0 && entryName[0] == '"' {
-			entry.name, err = strconv.Unquote(string(entryName))
-			if err != nil {
-				return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
-			}
-		} else {
-			entry.name = string(entryName)
+			return nil, err
 		}
+		entry.ptree = ptree
 
 		pos = posEnd + 1
 		entries = append(entries, entry)
@@ -100,7 +111,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.
 
 loop:
 	for sz > 0 {
-		mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
+		mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
 		if err != nil {
 			if err == io.EOF {
 				break loop

modules/git/pipeline/lfs_nogogit.go

@@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
 			case "tree":
 				var n int64
 				for n < size {
-					mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
+					mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
 					if err != nil {
 						return nil, err
 					}

modules/git/submodule.go

@@ -0,0 +1,63 @@
+package git
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+// SubModuleCommit submodule name and commit from a repository
+type SubModuleCommit struct {
+	Path   string
+	Commit string
+}
+
+// GetSubmoduleCommits returns a list of submodules paths and their commits from a repository
+// This function is only for generating new repos based on existing template, the template couldn't be too large.
+func GetSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []SubModuleCommit, _ error) {
+	stdoutReader, stdoutWriter, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	opts := &RunOpts{
+		Dir:    repoPath,
+		Stdout: stdoutWriter,
+		PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error {
+			_ = stdoutWriter.Close()
+			defer stdoutReader.Close()
+
+			scanner := bufio.NewScanner(stdoutReader)
+			for scanner.Scan() {
+				entry, err := parseLsTreeLine(scanner.Bytes())
+				if err != nil {
+					cancel()
+					return err
+				}
+				if entry.IsSubModule() {
+					submoduleCommits = append(submoduleCommits, SubModuleCommit{Path: entry.Name(), Commit: entry.ID.String()})
+				}
+			}
+			return scanner.Err()
+		},
+	}
+	err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts)
+	if err != nil {
+		return nil, fmt.Errorf("GetSubmoduleCommits: error running git ls-tree: %v", err)
+	}
+	return submoduleCommits, nil
+}
+
+// AddSubmoduleIndexes Adds the given submodules to the git index. Requires the .gitmodules file to be already present.
+func AddSubmoduleIndexes(ctx context.Context, repoPath string, submodules []SubModuleCommit) error {
+	for _, submodule := range submodules {
+		cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path)
+		if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil {
+			log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err)
+			return err
+		}
+	}
+	return nil
+}