Merge branch 'main' into chore/merge-main

.env.template

@@ -71,7 +71,7 @@ EMBEDDING_MODEL=text2vec
 #EMBEDDING_MODEL=bge-large-zh
 KNOWLEDGE_CHUNK_SIZE=500
 KNOWLEDGE_SEARCH_TOP_SIZE=5
-KNOWLEDGE_GRAPH_SEARCH_TOP_SIZE=50
+KNOWLEDGE_GRAPH_SEARCH_TOP_SIZE=200
 ## Maximum number of chunks to load at once, if your single document is too large,
 ## you can set this value to a higher value for better performance.
 ## if out of memory when load large document, you can set this value to a lower value.
@@ -157,6 +157,11 @@ EXECUTE_LOCAL_COMMANDS=False
 #*******************************************************************#
 VECTOR_STORE_TYPE=Chroma
 GRAPH_STORE_TYPE=TuGraph
+GRAPH_COMMUNITY_SUMMARY_ENABLED=True
+KNOWLEDGE_GRAPH_EXTRACT_SEARCH_TOP_SIZE=5
+KNOWLEDGE_GRAPH_EXTRACT_SEARCH_RECALL_SCORE=0.3
+KNOWLEDGE_GRAPH_COMMUNITY_SEARCH_TOP_SIZE=20
+KNOWLEDGE_GRAPH_COMMUNITY_SEARCH_RECALL_SCORE=0.0
 
 ### Chroma vector db config
 #CHROMA_PERSIST_PATH=/root/DB-GPT/pilot/data
@@ -187,7 +192,7 @@ ElasticSearch_PASSWORD={your_password}
 #TUGRAPH_PASSWORD=73@TuGraph
 #TUGRAPH_VERTEX_TYPE=entity
 #TUGRAPH_EDGE_TYPE=relation
-#TUGRAPH_EDGE_NAME_KEY=label
+#TUGRAPH_PLUGIN_NAMES=leiden
 
 #*******************************************************************#
 #**                  WebServer Language Support                   **#

.gitignore

@@ -4,7 +4,6 @@ __pycache__/
 *$py.class
 
 # C extensions
-*.so
 message/
 dbgpt/util/extensions/
 .env*
@@ -185,4 +184,4 @@ thirdparty
 /examples/**/*.gv
 /examples/**/*.gv.pdf
 /i18n/locales/**/**/*_ai_translated.po
-/i18n/locales/**/**/*~
+/i18n/locales/**/**/*~

dbgpt/_private/config.py

@@ -213,6 +213,9 @@ def __init__(self) -> None:
 
         # Vector Store Configuration
         self.VECTOR_STORE_TYPE = os.getenv("VECTOR_STORE_TYPE", "Chroma")
+        self.GRAPH_COMMUNITY_SUMMARY_ENABLED = (
+            os.getenv("GRAPH_COMMUNITY_SUMMARY_ENABLED", "").lower() == "true"
+        )
         self.MILVUS_URL = os.getenv("MILVUS_URL", "127.0.0.1")
         self.MILVUS_PORT = os.getenv("MILVUS_PORT", "19530")
         self.MILVUS_USERNAME = os.getenv("MILVUS_USERNAME", None)

dbgpt/agent/core/action/base.py

@@ -82,10 +82,10 @@ def to_dict(self) -> Dict[str, Any]:
 class Action(ABC, Generic[T]):
     """Base Action class for defining agent actions."""
 
-    def __init__(self):
+    def __init__(self, language: str = "en"):
         """Create an action."""
         self.resource: Optional[Resource] = None
-        self.language: str = "en"
+        self.language: str = language
 
     def init_resource(self, resource: Optional[Resource]):
         """Initialize the resource."""

dbgpt/agent/core/action/blank_action.py

@@ -12,9 +12,9 @@
 class BlankAction(Action):
     """Blank action class."""
 
-    def __init__(self):
-        """Create a blank action."""
-        super().__init__()
+    def __init__(self, **kwargs):
+        """Blank action init."""
+        super().__init__(**kwargs)
 
     @property
     def ai_out_schema(self) -> Optional[str]:

dbgpt/agent/core/agent_manage.py

@@ -127,17 +127,13 @@ def all_agents(self) -> Dict[str, str]:
     def list_agents(self):
         """Return a list of all registered agents and their descriptions."""
         result = []
-        from datetime import datetime
-
-        logger.info(f"List Agent Begin:{datetime.now()}")
         for name, value in self._agents.items():
             result.append(
                 {
                     "name": value[1].role,
                     "desc": value[1].goal,
                 }
             )
-        logger.info(f"List Agent End:{datetime.now()}")
         return result
 
 

dbgpt/agent/core/base_agent.py

@@ -673,7 +673,7 @@ def _init_actions(self, actions: List[Type[Action]]):
         self.actions = []
         for idx, action in enumerate(actions):
             if issubclass(action, Action):
-                self.actions.append(action())
+                self.actions.append(action(language=self.language))
 
     async def _a_append_message(
         self, message: AgentMessage, role, sender: Agent

dbgpt/agent/core/plan/plan_action.py

@@ -40,7 +40,7 @@ class PlanAction(Action[List[PlanInput]]):
 
     def __init__(self, **kwargs):
         """Create a plan action."""
-        super().__init__()
+        super().__init__(**kwargs)
         self._render_protocol = VisAgentPlans()
 
     @property

dbgpt/agent/expand/actions/chart_action.py

@@ -31,9 +31,9 @@ class SqlInput(BaseModel):
 class ChartAction(Action[SqlInput]):
     """Chart action class."""
 
-    def __init__(self):
-        """Create a chart action."""
-        super().__init__()
+    def __init__(self, **kwargs):
+        """Chart action init."""
+        super().__init__(**kwargs)
         self._render_protocol = VisChart()
 
     @property

dbgpt/agent/expand/actions/code_action.py

@@ -16,9 +16,9 @@
 class CodeAction(Action[None]):
     """Code Action Module."""
 
-    def __init__(self):
-        """Create a code action."""
-        super().__init__()
+    def __init__(self, **kwargs):
+        """Code action init."""
+        super().__init__(**kwargs)
         self._render_protocol = VisCode()
         self._code_execution_config = {}
 

dbgpt/agent/expand/actions/dashboard_action.py

@@ -39,9 +39,9 @@ def to_dict(self):
 class DashboardAction(Action[List[ChartItem]]):
     """Dashboard action class."""
 
-    def __init__(self):
-        """Create a dashboard action."""
-        super().__init__()
+    def __init__(self, **kwargs):
+        """Dashboard action init."""
+        super().__init__(**kwargs)
         self._render_protocol = VisDashboard()
 
     @property

dbgpt/agent/expand/actions/indicator_action.py

@@ -41,9 +41,9 @@ class IndicatorInput(BaseModel):
 class IndicatorAction(Action[IndicatorInput]):
     """Indicator Action."""
 
-    def __init__(self):
-        """Init Indicator Action."""
-        super().__init__()
+    def __init__(self, **kwargs):
+        """Init indicator action."""
+        super().__init__(**kwargs)
         self._render_protocol = VisApiResponse()
 
     @property

dbgpt/agent/expand/actions/tool_action.py

@@ -34,9 +34,9 @@ class ToolInput(BaseModel):
 class ToolAction(Action[ToolInput]):
     """Tool action class."""
 
-    def __init__(self):
-        """Create a plugin action."""
-        super().__init__()
+    def __init__(self, **kwargs):
+        """Tool action init."""
+        super().__init__(**kwargs)
         self._render_protocol = VisPlugin()
 
     @property

dbgpt/agent/resource/knowledge.py

@@ -73,8 +73,8 @@ async def get_prompt(
         prompt_template = f"\nResources-{self.name}:\n {content}"
         prompt_template_zh = f"\n资源-{self.name}:\n {content}"
         if lang == "en":
-            return prompt_template.format(content=content), self._get_references(chunks)
-        return prompt_template_zh.format(content=content), self._get_references(chunks)
+            return prompt_template, self._get_references(chunks)
+        return prompt_template_zh, self._get_references(chunks)
 
     async def get_resources(
         self,

dbgpt/app/knowledge/api.py

@@ -112,13 +112,15 @@ def arguments(space_id: str):
 
 
 @router.post("/knowledge/{space_name}/recall_test")
-def recall_test(
+async def recall_test(
     space_name: str,
     request: DocumentRecallTestRequest,
 ):
     print(f"/knowledge/{space_name}/recall_test params:")
     try:
-        return Result.succ(knowledge_space_service.recall_test(space_name, request))
+        return Result.succ(
+            await knowledge_space_service.recall_test(space_name, request)
+        )
     except Exception as e:
         return Result.failed(code="E000X", msg=f"{space_name} recall_test error {e}")
 

dbgpt/app/knowledge/service.py

@@ -309,7 +309,7 @@ def get_knowledge_space_by_ids(self, ids):
         """
         return knowledge_space_dao.get_knowledge_space_by_ids(ids)
 
-    def recall_test(
+    async def recall_test(
         self, space_name, doc_recall_test_request: DocumentRecallTestRequest
     ):
         logger.info(f"recall_test {space_name}, {doc_recall_test_request}")
@@ -338,7 +338,7 @@ def recall_test(
             knowledge_space_retriever = KnowledgeSpaceRetriever(
                 space_id=space.id, top_k=top_k
             )
-            chunks = knowledge_space_retriever.retrieve_with_scores(
+            chunks = await knowledge_space_retriever.aretrieve_with_scores(
                 question, score_threshold
             )
             retrievers_end_time = timeit.default_timer()
@@ -646,13 +646,16 @@ def query_graph(self, space_name, limit):
         graph = vector_store_connector.client.query_graph(limit=limit)
         res = {"nodes": [], "edges": []}
         for node in graph.vertices():
-            res["nodes"].append({"vid": node.vid})
-        for edge in graph.edges():
-            res["edges"].append(
+            res["nodes"].append(
                 {
-                    "src": edge.sid,
-                    "dst": edge.tid,
-                    "label": edge.props[graph.edge_label],
+                    "id": node.vid,
+                    "communityId": node.get_prop("_community_id"),
+                    "name": node.vid,
+                    "type": "",
                 }
             )
+        for edge in graph.edges():
+            res["edges"].append(
+                {"source": edge.sid, "target": edge.tid, "name": edge.name, "type": ""}
+            )
         return res

dbgpt/app/openapi/api_v1/api_v1.py

@@ -548,6 +548,17 @@ async def chat_completions(
                     headers=headers,
                     media_type="text/plain",
                 )
+    except Exception as e:
+        logger.exception(f"Chat Exception!{dialogue}", e)
+
+        async def error_text(err_msg):
+            yield f"data:{err_msg}\n\n"
+
+        return StreamingResponse(
+            error_text(str(e)),
+            headers=headers,
+            media_type="text/plain",
+        )
     finally:
         # write to recent usage app.
         if dialogue.user_name is not None and dialogue.app_code is not None:

dbgpt/app/scene/chat_data/chat_excel/excel_analyze/chat.py

@@ -34,6 +34,8 @@ def __init__(self, chat_param: Dict):
         """
 
         self.select_param = chat_param["select_param"]
+        if not self.select_param:
+            raise ValueError("Please upload the Excel document you want to talk to！")
         self.model_name = chat_param["model_name"]
         chat_param["chat_mode"] = ChatScene.ChatExcel
         self.chat_param = chat_param

dbgpt/app/scene/chat_data/chat_excel/excel_reader.py

@@ -230,7 +230,7 @@ def is_chinese(text):
 
 
 class ExcelReader:
-    def __init__(self, conv_uid, file_param):
+    def __init__(self, conv_uid: str, file_param: str):
         self.conv_uid = conv_uid
         self.file_param = file_param
         if isinstance(file_param, str) and os.path.isabs(file_param):

dbgpt/app/static/web/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="icon" href="/favicon.ico"/><meta name="description" content="Revolutionizing Database Interactions with Private LLM Technology"/><meta property="og:description" content="eosphoros-ai"/><meta property="og:title" content="DB-GPT"/><link rel="preload" href="/_next/static/css/f8eb45c952dd19e2.css" as="style"/><link rel="stylesheet" href="/_next/static/css/f8eb45c952dd19e2.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-f159616fcd1bda6b.js" defer=""></script><script src="/_next/static/chunks/framework-bf941633d42c5f92.js" defer=""></script><script src="/_next/static/chunks/main-28c79a921c889131.js" defer=""></script><script src="/_next/static/chunks/pages/_app-690c49bd776698cb.js" defer=""></script><script src="/_next/static/chunks/pages/_error-8095ba9e1bf12f30.js" defer=""></script><script src="/_next/static/_LUlP8fg3TFDUAFikQDdn/_buildManifest.js" defer=""></script><script src="/_next/static/_LUlP8fg3TFDUAFikQDdn/_ssgManifest.js" defer=""></script><link rel="stylesheet" href="/_next/static/css/antd-output/antd.min.7d5365b5.css"/></head><body><div id="__next"><div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"_LUlP8fg3TFDUAFikQDdn","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="icon" href="/favicon.ico"/><meta name="description" content="Revolutionizing Database Interactions with Private LLM Technology"/><meta property="og:description" content="eosphoros-ai"/><meta property="og:title" content="DB-GPT"/><link rel="preload" href="/_next/static/css/9688596e74821727.css" as="style"/><link rel="stylesheet" href="/_next/static/css/9688596e74821727.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-ec1d3d859d29d7e0.js" defer=""></script><script src="/_next/static/chunks/framework-8b06d32cbb857e0e.js" defer=""></script><script src="/_next/static/chunks/main-6c4c7f5b8c9b1320.js" defer=""></script><script src="/_next/static/chunks/pages/_app-d05b54d40f29633d.js" defer=""></script><script src="/_next/static/chunks/pages/_error-8095ba9e1bf12f30.js" defer=""></script><script src="/_next/static/60Qx1TLPG2YRtycocqSLP/_buildManifest.js" defer=""></script><script src="/_next/static/60Qx1TLPG2YRtycocqSLP/_ssgManifest.js" defer=""></script><link rel="stylesheet" href="/_next/static/css/antd-output/antd.min.7d5365b5.css"/></head><body><div id="__next"><div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"60Qx1TLPG2YRtycocqSLP","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

dbgpt/app/static/web/404/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="icon" href="/favicon.ico"/><meta name="description" content="Revolutionizing Database Interactions with Private LLM Technology"/><meta property="og:description" content="eosphoros-ai"/><meta property="og:title" content="DB-GPT"/><link rel="preload" href="/_next/static/css/f8eb45c952dd19e2.css" as="style"/><link rel="stylesheet" href="/_next/static/css/f8eb45c952dd19e2.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-f159616fcd1bda6b.js" defer=""></script><script src="/_next/static/chunks/framework-bf941633d42c5f92.js" defer=""></script><script src="/_next/static/chunks/main-28c79a921c889131.js" defer=""></script><script src="/_next/static/chunks/pages/_app-690c49bd776698cb.js" defer=""></script><script src="/_next/static/chunks/pages/_error-8095ba9e1bf12f30.js" defer=""></script><script src="/_next/static/_LUlP8fg3TFDUAFikQDdn/_buildManifest.js" defer=""></script><script src="/_next/static/_LUlP8fg3TFDUAFikQDdn/_ssgManifest.js" defer=""></script><link rel="stylesheet" href="/_next/static/css/antd-output/antd.min.7d5365b5.css"/></head><body><div id="__next"><div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"_LUlP8fg3TFDUAFikQDdn","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="icon" href="/favicon.ico"/><meta name="description" content="Revolutionizing Database Interactions with Private LLM Technology"/><meta property="og:description" content="eosphoros-ai"/><meta property="og:title" content="DB-GPT"/><link rel="preload" href="/_next/static/css/9688596e74821727.css" as="style"/><link rel="stylesheet" href="/_next/static/css/9688596e74821727.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-ec1d3d859d29d7e0.js" defer=""></script><script src="/_next/static/chunks/framework-8b06d32cbb857e0e.js" defer=""></script><script src="/_next/static/chunks/main-6c4c7f5b8c9b1320.js" defer=""></script><script src="/_next/static/chunks/pages/_app-d05b54d40f29633d.js" defer=""></script><script src="/_next/static/chunks/pages/_error-8095ba9e1bf12f30.js" defer=""></script><script src="/_next/static/60Qx1TLPG2YRtycocqSLP/_buildManifest.js" defer=""></script><script src="/_next/static/60Qx1TLPG2YRtycocqSLP/_ssgManifest.js" defer=""></script><link rel="stylesheet" href="/_next/static/css/antd-output/antd.min.7d5365b5.css"/></head><body><div id="__next"><div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"60Qx1TLPG2YRtycocqSLP","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>