Merge branch 'main' into couchintegration

Author: crazywoola

api/controllers/console/workspace/model_providers.py

@@ -126,13 +126,12 @@ class ModelProviderIconApi(Resource):
     Get model provider icon
     """
 
-    @setup_required
-    @login_required
-    @account_initialization_required
     def get(self, provider: str, icon_type: str, lang: str):
         model_provider_service = ModelProviderService()
         icon, mimetype = model_provider_service.get_model_provider_icon(
-            provider=provider, icon_type=icon_type, lang=lang
+            provider=provider,
+            icon_type=icon_type,
+            lang=lang,
         )
 
         return send_file(io.BytesIO(icon), mimetype=mimetype)

api/core/model_runtime/model_providers/siliconflow/llm/llm.py

@@ -1,8 +1,18 @@
 from collections.abc import Generator
 from typing import Optional, Union
 
-from core.model_runtime.entities.llm_entities import LLMResult
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult
 from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
+from core.model_runtime.entities.model_entities import (
+    AIModelEntity,
+    FetchFrom,
+    ModelFeature,
+    ModelPropertyKey,
+    ModelType,
+    ParameterRule,
+    ParameterType,
+)
 from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAIAPICompatLargeLanguageModel
 
 
@@ -29,3 +39,53 @@ def validate_credentials(self, model: str, credentials: dict) -> None:
     def _add_custom_parameters(cls, credentials: dict) -> None:
         credentials["mode"] = "chat"
         credentials["endpoint_url"] = "https://api.siliconflow.cn/v1"
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None:
+        return AIModelEntity(
+            model=model,
+            label=I18nObject(en_US=model, zh_Hans=model),
+            model_type=ModelType.LLM,
+            features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL, ModelFeature.STREAM_TOOL_CALL]
+            if credentials.get("function_calling_type") == "tool_call"
+            else [],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 8000)),
+                ModelPropertyKey.MODE: LLMMode.CHAT.value,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name="temperature",
+                    use_template="temperature",
+                    label=I18nObject(en_US="Temperature", zh_Hans="温度"),
+                    type=ParameterType.FLOAT,
+                ),
+                ParameterRule(
+                    name="max_tokens",
+                    use_template="max_tokens",
+                    default=512,
+                    min=1,
+                    max=int(credentials.get("max_tokens", 1024)),
+                    label=I18nObject(en_US="Max Tokens", zh_Hans="最大标记"),
+                    type=ParameterType.INT,
+                ),
+                ParameterRule(
+                    name="top_p",
+                    use_template="top_p",
+                    label=I18nObject(en_US="Top P", zh_Hans="Top P"),
+                    type=ParameterType.FLOAT,
+                ),
+                ParameterRule(
+                    name="top_k",
+                    use_template="top_k",
+                    label=I18nObject(en_US="Top K", zh_Hans="Top K"),
+                    type=ParameterType.FLOAT,
+                ),
+                ParameterRule(
+                    name="frequency_penalty",
+                    use_template="frequency_penalty",
+                    label=I18nObject(en_US="Frequency Penalty", zh_Hans="重复惩罚"),
+                    type=ParameterType.FLOAT,
+                ),
+            ],
+        )

api/core/model_runtime/model_providers/siliconflow/siliconflow.yaml

@@ -20,6 +20,7 @@ supported_model_types:
   - speech2text
 configurate_methods:
   - predefined-model
+  - customizable-model
 provider_credential_schema:
   credential_form_schemas:
     - variable: api_key
@@ -30,3 +31,57 @@ provider_credential_schema:
       placeholder:
         zh_Hans: 在此输入您的 API Key
         en_US: Enter your API Key
+model_credential_schema:
+  model:
+    label:
+      en_US: Model Name
+      zh_Hans: 模型名称
+    placeholder:
+      en_US: Enter your model name
+      zh_Hans: 输入模型名称
+  credential_form_schemas:
+    - variable: api_key
+      label:
+        en_US: API Key
+      type: secret-input
+      required: true
+      placeholder:
+        zh_Hans: 在此输入您的 API Key
+        en_US: Enter your API Key
+    - variable: context_size
+      label:
+        zh_Hans: 模型上下文长度
+        en_US: Model context size
+      required: true
+      type: text-input
+      default: '4096'
+      placeholder:
+        zh_Hans: 在此输入您的模型上下文长度
+        en_US: Enter your Model context size
+    - variable: max_tokens
+      label:
+        zh_Hans: 最大 token 上限
+        en_US: Upper bound for max tokens
+      default: '4096'
+      type: text-input
+      show_on:
+        - variable: __model_type
+          value: llm
+    - variable: function_calling_type
+      label:
+        en_US: Function calling
+      type: select
+      required: false
+      default: no_call
+      options:
+        - value: no_call
+          label:
+            en_US: Not Support
+            zh_Hans: 不支持
+        - value: function_call
+          label:
+            en_US: Support
+            zh_Hans: 支持
+      show_on:
+        - variable: __model_type
+          value: llm

api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import math
 from typing import Any, Optional
 from urllib.parse import urlparse
 
@@ -112,7 +113,8 @@ def delete(self) -> None:
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
         top_k = kwargs.get("top_k", 10)
-        knn = {"field": Field.VECTOR.value, "query_vector": query_vector, "k": top_k}
+        num_candidates = math.ceil(top_k * 1.5)
+        knn = {"field": Field.VECTOR.value, "query_vector": query_vector, "k": top_k, "num_candidates": num_candidates}
 
         results = self._client.search(index=self._collection_name, knn=knn, size=top_k)
 

api/services/dataset_service.py

@@ -8,6 +8,7 @@
 
 from flask_login import current_user
 from sqlalchemy import func
+from werkzeug.exceptions import NotFound
 
 from configs import dify_config
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
@@ -975,6 +976,8 @@ def update_document_with_dataset_id(
     ):
         DatasetService.check_dataset_model_setting(dataset)
         document = DocumentService.get_document(dataset.id, document_data["original_document_id"])
+        if document is None:
+            raise NotFound("Document not found")
         if document.display_status != "available":
             raise ValueError("Document is not available")
         # update document name

api/services/enterprise/base.py

@@ -7,11 +7,16 @@ class EnterpriseRequest:
     base_url = os.environ.get("ENTERPRISE_API_URL", "ENTERPRISE_API_URL")
     secret_key = os.environ.get("ENTERPRISE_API_SECRET_KEY", "ENTERPRISE_API_SECRET_KEY")
 
+    proxies = {
+        "http": None,
+        "https": None,
+    }
+
     @classmethod
     def send_request(cls, method, endpoint, json=None, params=None):
         headers = {"Content-Type": "application/json", "Enterprise-Api-Secret-Key": cls.secret_key}
 
         url = f"{cls.base_url}{endpoint}"
-        response = requests.request(method, url, json=json, params=params, headers=headers)
+        response = requests.request(method, url, json=json, params=params, headers=headers, proxies=cls.proxies)
 
         return response.json()

api/tests/artifact_tests/dependencies/test_dependencies_sorted.py

@@ -2,36 +2,23 @@
 
 import toml
 
-ALL_DEPENDENCY_GROUP_NAMES = [
-    # default main group
-    "",
-    # required groups
-    "indirect",
-    "storage",
-    "tools",
-    "vdb",
-    # optional groups
-    "dev",
-    "lint",
-]
-
 
 def load_api_poetry_configs() -> dict[str, Any]:
     pyproject_toml = toml.load("api/pyproject.toml")
-    return pyproject_toml.get("tool").get("poetry")
+    return pyproject_toml["tool"]["poetry"]
 
 
-def load_dependency_groups() -> dict[str, dict[str, dict[str, Any]]]:
-    poetry_configs = load_api_poetry_configs()
-    group_name_to_dependencies = {
-        group_name: (poetry_configs.get("group").get(group_name) if group_name else poetry_configs).get("dependencies")
-        for group_name in ALL_DEPENDENCY_GROUP_NAMES
-    }
-    return group_name_to_dependencies
+def load_all_dependency_groups() -> dict[str, dict[str, dict[str, Any]]]:
+    configs = load_api_poetry_configs()
+    configs_by_group = {"main": configs}
+    for group_name in configs["group"]:
+        configs_by_group[group_name] = configs["group"][group_name]
+    dependencies_by_group = {group_name: base["dependencies"] for group_name, base in configs_by_group.items()}
+    return dependencies_by_group
 
 
 def test_group_dependencies_sorted():
-    for group_name, dependencies in load_dependency_groups().items():
+    for group_name, dependencies in load_all_dependency_groups().items():
         dependency_names = list(dependencies.keys())
         expected_dependency_names = sorted(set(dependency_names))
         section = f"tool.poetry.group.{group_name}.dependencies" if group_name else "tool.poetry.dependencies"
@@ -42,17 +29,18 @@ def test_group_dependencies_sorted():
 
 
 def test_group_dependencies_version_operator():
-    for group_name, dependencies in load_dependency_groups().items():
+    for group_name, dependencies in load_all_dependency_groups().items():
         for dependency_name, specification in dependencies.items():
-            version_spec = specification if isinstance(specification, str) else specification.get("version")
+            version_spec = specification if isinstance(specification, str) else specification["version"]
             assert not version_spec.startswith("^"), (
-                f"'^' is not allowed in dependency version," f" but found in '{dependency_name} = {version_spec}'"
+                f"Please replace '{dependency_name} = {version_spec}' with '{dependency_name} = ~{version_spec[1:]}' "
+                f"'^' operator is too wide and not allowed in the version specification."
             )
 
 
 def test_duplicated_dependency_crossing_groups():
     all_dependency_names: list[str] = []
-    for dependencies in load_dependency_groups().values():
+    for dependencies in load_all_dependency_groups().values():
         dependency_names = list(dependencies.keys())
         all_dependency_names.extend(dependency_names)
     expected_all_dependency_names = set(all_dependency_names)

docker/.env.example

@@ -804,3 +804,5 @@ POSITION_TOOL_EXCLUDES=
 POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
+# CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
+CSP_WHITELIST=