Merge pull request #3011 from actiontech/support_git_clone_protocol

Support git clone protocol

Author: iwanghc

sqle/api/controller/v1/configuration.go

@@ -1,11 +1,14 @@
 package v1
 
 import (
+	"context"
 	"crypto/rsa"
 	"fmt"
 	"net/http"
+	"os"
 	"strconv"
 
+	"github.com/actiontech/sqle/sqle/errors"
 	"github.com/actiontech/sqle/sqle/utils"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/storer"
@@ -15,6 +18,11 @@ import (
 
 	"github.com/actiontech/sqle/sqle/model"
 
+	goGit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	goGitTransport "github.com/go-git/go-git/v5/plumbing/transport/http"
+
+	sshTransport "github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/labstack/echo/v4"
 	"golang.org/x/crypto/ssh"
 )
@@ -442,7 +450,7 @@ func testGitConnectionV1(c echo.Context) error {
 	if err := controller.BindAndValidateReq(c, request); err != nil {
 		return controller.JSONBaseErrorReq(c, err)
 	}
-	repository, _, cleanup, err := utils.CloneGitRepository(c.Request().Context(), request.GitHttpUrl, request.GitUserName, request.GitUserPassword)
+	repository, _, cleanup, err := CloneGitRepository(c.Request().Context(), request.GitHttpUrl, request.GitUserName, request.GitUserPassword, "")
 	if err != nil {
 		return c.JSON(http.StatusOK, &TestGitConnectionResV1{
 			BaseRes: controller.NewBaseReq(nil),
@@ -469,6 +477,15 @@ func testGitConnectionV1(c echo.Context) error {
 		})
 	}
 	branches, err := getBranches(references)
+	if err != nil {
+		return c.JSON(http.StatusOK, &TestGitConnectionResV1{
+			BaseRes: controller.NewBaseReq(nil),
+			Data: TestGitConnectionResDataV1{
+				IsConnectedSuccess: false,
+				ErrorMessage:       err.Error(),
+			},
+		})
+	}
 	return c.JSON(http.StatusOK, &TestGitConnectionResV1{
 		BaseRes: controller.NewBaseReq(nil),
 		Data: TestGitConnectionResDataV1{
@@ -478,6 +495,82 @@ func testGitConnectionV1(c echo.Context) error {
 	})
 }
 
+func CloneGitRepository(ctx context.Context, url, username, password, branch string) (repository *goGit.Repository, directory string, cleanup func() error, err error) {
+	// 创建一个临时目录用于存放克隆的仓库
+	directory, err = os.MkdirTemp("./", "git-repo-")
+	if err != nil {
+		return nil, "", nil, err
+	}
+	// 定义清理函数，用于删除临时目录
+	cleanup = func() error {
+		return os.RemoveAll(directory)
+	}
+
+	cloneOpts := &goGit.CloneOptions{
+		URL: url,
+	}
+	// TODO use branch name to clone single branch on the repo
+	// if branch != "" {
+	// cloneOpts.ReferenceName = plumbing.ReferenceName(branch)
+	// }
+	ep, err := transport.NewEndpoint(url)
+	if err != nil {
+		return nil, "", nil, err
+	}
+
+	switch {
+	case ep.Protocol == "http" || ep.Protocol == "https":
+		// http协议下：
+		//   1. 账号密码登录
+		//       username/password
+		//   2. token 方式
+		//    对于 GitLab/github，用户名可以是任意非空字符串,建议填oauth2
+		//       gitlab：
+		// 			oauth2/token
+		//       github:
+		// 			oauth2/token
+
+		cloneOpts.Auth = &goGitTransport.BasicAuth{
+			Username: username,
+			Password: password,
+		}
+	case ep.Protocol == "ssh":
+		// ssh 协议
+		//     	前置条件：
+		// 			1. 生成密钥
+		// 			2. 查看公钥
+		// 			3. 仓库配置密钥
+		//             不支持该步骤，用户手动执行
+		storage := model.GetStorage()
+		privateKey, exists, err := storage.GetSystemVariableByKey(model.SystemVariableSSHPrimaryKey)
+		if err != nil {
+			return nil, directory, cleanup, err
+		}
+		if !exists {
+			return nil, directory, cleanup, errors.New(errors.DataNotExist, fmt.Errorf("git ssh private key not found"))
+		}
+		publicKeys, err := sshTransport.NewPublicKeys("git", []byte(privateKey.Value), "")
+		if err != nil {
+			return nil, directory, cleanup, fmt.Errorf("failed to load SSH key: %w", err)
+		}
+		publicKeys.HostKeyCallback = ssh.InsecureIgnoreHostKey()
+		cloneOpts.Auth = publicKeys
+	case ep.Protocol == "git":
+		// git协议
+		// 不需要校验权限
+	// case  IsFile TODO
+	default:
+		return nil, "", cleanup, errors.New(errors.DataInvalid, fmt.Errorf("url is not a git url"))
+	}
+
+	repository, err = goGit.PlainCloneContext(ctx, directory, false, cloneOpts)
+	if err != nil {
+		return nil, directory, cleanup, err
+	}
+
+	return repository, directory, cleanup, nil
+}
+
 func getBranches(references storer.ReferenceIter) ([]string, error) {
 	branches := make([]string, 0)
 	err := references.ForEach(func(ref *plumbing.Reference) error {
@@ -558,18 +651,24 @@ type SSHPublicKeyInfo struct {
 // @Tags configuration
 // @Id getSSHPublicKey
 // @Security ApiKeyAuth
-// @Success 200 {object} v1.SSHPublicKeyInfo
+// @Success 200 {object} v1.SSHPublicKeyInfoV1Rsp
 // @Router /v1/configurations/ssh_key [get]
 func GetSSHPublicKey(c echo.Context) error {
 	storage := model.GetStorage()
 	systemVariables, exists, err := storage.GetSystemVariableByKey(model.SystemVariableSSHPrimaryKey)
 	if err != nil {
-		c.Logger().Errorf("failed to get ssh public key: %v", err)
 		return controller.JSONBaseErrorReq(c, err)
 	}
+	if !exists {
+		return c.JSON(http.StatusOK, SSHPublicKeyInfoV1Rsp{
+			BaseRes: controller.NewBaseReq(nil),
+			Data: SSHPublicKeyInfo{
+				PublicKey: "",
+			},
+		})
+	}
 	privateKey, err := ssh.ParseRawPrivateKey([]byte(systemVariables.Value))
 	if err != nil {
-		c.Logger().Errorf("failed to parse SSH private key: %v", err)
 		return controller.JSONBaseErrorReq(c, err)
 	}
 	rsaPrivateKey, ok := privateKey.(*rsa.PrivateKey)
@@ -584,12 +683,7 @@ func GetSSHPublicKey(c echo.Context) error {
 	return c.JSON(http.StatusOK,
 		SSHPublicKeyInfoV1Rsp{
 			Data: SSHPublicKeyInfo{
-				PublicKey: func() string {
-					if !exists {
-						return ""
-					}
-					return publicKeyStr
-				}(),
+				PublicKey: publicKeyStr,
 			},
 		})
 }

sqle/api/controller/v1/sql_audit_record.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	e "errors"
 	"fmt"
-	"github.com/go-git/go-git/v5/plumbing"
 	"io"
 	"io/fs"
 	"net/http"
@@ -15,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-git/go-git/v5/plumbing"
+
 	javaParser "github.com/actiontech/java-sql-extractor/parser"
 	xmlParser "github.com/actiontech/mybatis-mapper-2-sql"
 	"github.com/actiontech/sqle/sqle/api/controller"
@@ -441,7 +442,7 @@ func parseXMLsWithFilePath(xmlContents []xmlParser.XmlFile) ([]SQLFromXML, error
 // todo 此处跳过了不支持的编码格式文件
 func getSqlsFromGit(c echo.Context) (sqlsFromSQLFiles, sqlsFromJavaFiles []SQLsFromSQLFile, sqlsFromXMLs []SQLFromXML, exist bool, err error) {
 	// clone from git
-	repository, directory, cleanup, err := utils.CloneGitRepository(c.Request().Context(), c.FormValue(GitHttpURL), c.FormValue(GitUserName), c.FormValue(GitPassword))
+	repository, directory, cleanup, err := CloneGitRepository(c.Request().Context(), c.FormValue(GitHttpURL), c.FormValue(GitUserName), c.FormValue(GitPassword), c.FormValue(GitBranchName))
 	if err != nil {
 		return nil, nil, nil, false, err
 	}

sqle/utils/util.go

@@ -2,7 +2,6 @@ package utils
 
 import (
 	"bytes"
-	"context"
 	"crypto/md5"
 	"crypto/rand"
 	"crypto/rsa"
@@ -15,7 +14,6 @@ import (
 	"io"
 	"math"
 	"net/url"
-	"os"
 	"regexp"
 	"sort"
 	"strconv"
@@ -26,9 +24,6 @@ import (
 	"unicode/utf8"
 	"unsafe"
 
-	sqleErrors "github.com/actiontech/sqle/sqle/errors"
-	goGit "github.com/go-git/go-git/v5"
-	goGitTransport "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"golang.org/x/crypto/ssh"
 
 	"github.com/actiontech/sqle/sqle/log"
@@ -464,37 +459,6 @@ func IntersectionStringSlice(slice1, slice2 []string) []string {
 	}
 	return intersection
 }
-
-func CloneGitRepository(ctx context.Context, url, username, password string) (repository *goGit.Repository, directory string, cleanup func() error, err error) {
-	if !IsGitHttpURL(url) {
-		return nil, "", nil, sqleErrors.New(sqleErrors.DataInvalid, fmt.Errorf("url is not a git url"))
-	}
-	// 创建一个临时目录用于存放克隆的仓库
-	directory, err = os.MkdirTemp("./", "git-repo-")
-	if err != nil {
-		return nil, "", nil, err
-	}
-	// 定义清理函数，用于删除临时目录
-	cleanup = func() error {
-		return os.RemoveAll(directory)
-	}
-	cloneOpts := &goGit.CloneOptions{
-		URL: url,
-	}
-	if username != "" {
-		cloneOpts.Auth = &goGitTransport.BasicAuth{
-			Username: username,
-			Password: password,
-		}
-	}
-	repository, err = goGit.PlainCloneContext(ctx, directory, false, cloneOpts)
-	if err != nil {
-		err = cleanup()
-		return nil, directory, nil, err
-	}
-	return repository, directory, cleanup, nil
-}
-
 func GeneratePublicKeyFromPrivateKey(privateKey *rsa.PrivateKey) (string, error) {
 	publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey)
 	if err != nil {