feat: gemini模型适配图片对话

Author: bincooo

internal/common/messages.go

@@ -182,7 +182,7 @@ func TextMessageCombiner[T any](
 			Message  map[string]string
 			Buffer   *bytes.Buffer
 			Initial  func() pkg.Keyv[interface{}]
-		}{Previous: previous, Next: previous, Message: message, Buffer: buffer, Initial: func() pkg.Keyv[interface{}] {
+		}{Previous: previous, Next: next, Message: message, Buffer: buffer, Initial: func() pkg.Keyv[interface{}] {
 			if id, ok := message["id"]; ok {
 				return sources[id]
 			}
@@ -193,10 +193,7 @@ func TextMessageCombiner[T any](
 			return nil, err
 		}
 
-		if len(next) > 0 {
-			newMessages = append(newMessages, nextMessages...)
-		}
-
+		newMessages = append(newMessages, nextMessages...)
 		previous = message["role"]
 	}
 	return

internal/plugin/llm/gemini/adapter.go

@@ -79,7 +79,12 @@ func (API) Completion(ctx *gin.Context) {
 		matchers   = com.GetGinMatchers(ctx)
 	)
 
-	newMessages, tokens := mergeMessages(completion.Messages)
+	newMessages, tokens, err := mergeMessages(completion.Messages)
+	if err != nil {
+		response.Error(ctx, -1, err)
+		return
+	}
+
 	ctx.Set(ginTokens, tokens)
 	r, err := build(ctx.Request.Context(), proxies, cookie, newMessages, completion)
 	if err != nil {

internal/plugin/llm/gemini/fetch.go

@@ -187,9 +187,9 @@ func build(ctx context.Context, proxies, token string, messages []map[string]int
 	if res.StatusCode != http.StatusOK {
 		h := res.Header
 		if c := h.Get("content-type"); !strings.Contains(c, "text/html") {
-			bts, e := io.ReadAll(res.Body)
+			dataBytes, e := io.ReadAll(res.Body)
 			if e == nil {
-				logger.Errorf("%s", bts)
+				logger.Errorf("%s", dataBytes)
 			}
 		}
 		return nil, errors.New(res.Status)

internal/plugin/llm/gemini/message.go

@@ -122,7 +122,7 @@ func waitResponse(ctx *gin.Context, matchers []com.Matcher, partialResponse *htt
 	}
 }
 
-func mergeMessages(messages []pkg.Keyv[interface{}]) (newMessages []map[string]interface{}, tokens int) {
+func mergeMessages(messages []pkg.Keyv[interface{}]) (newMessages []map[string]interface{}, tokens int, err error) {
 	// role类型转换
 	condition := func(expr string) string {
 		switch expr {
@@ -195,6 +195,55 @@ func mergeMessages(messages []pkg.Keyv[interface{}]) (newMessages []map[string]i
 			return
 		}
 
+		// 复合消息
+		if _, ok := opts.Message["multi"]; ok && role == "user" {
+			message := opts.Initial()
+			values := message.GetSlice("content")
+			if len(values) == 0 {
+				return
+			}
+
+			var multi []interface{}
+			for _, value := range values {
+				var keyv pkg.Keyv[interface{}]
+				keyv, ok = value.(map[string]interface{})
+				if !ok {
+					continue
+				}
+
+				if keyv.Is("type", "text") {
+					multi = append(multi, map[string]interface{}{
+						"text": keyv.GetString("text"),
+					})
+				}
+
+				if keyv.Is("type", "image_url") {
+					o := keyv.GetKeyv("image_url")
+					mime, data, e := com.LoadImageMeta(o.GetString("url"))
+					if e != nil {
+						err = e
+						return
+					}
+					multi = append(multi, map[string]interface{}{
+						"inlineData": map[string]interface{}{
+							"mimeType": mime,
+							"data":     data,
+						},
+					})
+				}
+			}
+
+			if len(multi) == 0 {
+				return
+			}
+
+			result = append(result, map[string]interface{}{
+				"role":  condition("user"),
+				"parts": multi,
+			})
+			return
+		}
+
 		if role == "system" {
 			result = append(result, map[string]interface{}{
 				"role": "user",
@@ -228,6 +277,6 @@ func mergeMessages(messages []pkg.Keyv[interface{}]) (newMessages []map[string]i
 		return
 	}
 
-	newMessages, _ = com.TextMessageCombiner(messages, iterator)
+	newMessages, err = com.TextMessageCombiner(messages, iterator)
 	return
 }