From 782b9a79035fdb03fdc11acd7e3c7e2347e3543f Mon Sep 17 00:00:00 2001 From: Vadym Barda Date: Fri, 1 Nov 2024 17:40:53 -0400 Subject: [PATCH] langgraph: add message list validation to create_react_agent + a troubleshooting guide (#2182) --- docs/docs/how-tos/index.md | 3 +- .../errors/INVALID_CHAT_HISTORY.md | 30 +++++++++ docs/docs/troubleshooting/errors/index.md | 1 + libs/langgraph/langgraph/errors.py | 1 + .../langgraph/prebuilt/chat_agent_executor.py | 34 ++++++++++ libs/langgraph/tests/test_prebuilt.py | 66 +++++++++++++++++++ 6 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 docs/docs/troubleshooting/errors/INVALID_CHAT_HISTORY.md diff --git a/docs/docs/how-tos/index.md b/docs/docs/how-tos/index.md index c19007b2c..4689ebb0a 100644 --- a/docs/docs/how-tos/index.md +++ b/docs/docs/how-tos/index.md @@ -220,11 +220,12 @@ LangGraph Studio is a built-in UI for visualizing, testing, and debugging your a ## Troubleshooting -The [Error Reference](../troubleshooting/errors/index.md) page contains guides around resolving common errors you may find while building with LangGraph. Errors referenced below will have an `lc_error_code` property corresponding to one of the below codes when they are thrown in code. +These are the guides for resolving common errors you may find while building with LangGraph. Errors referenced below will have an `lc_error_code` property corresponding to one of the below codes when they are thrown in code. - [GRAPH_RECURSION_LIMIT](../troubleshooting/errors/GRAPH_RECURSION_LIMIT.md) - [INVALID_CONCURRENT_GRAPH_UPDATE](../troubleshooting/errors/INVALID_CONCURRENT_GRAPH_UPDATE.md) - [INVALID_GRAPH_NODE_RETURN_VALUE](../troubleshooting/errors/INVALID_GRAPH_NODE_RETURN_VALUE.md) - [MULTIPLE_SUBGRAPHS](../troubleshooting/errors/MULTIPLE_SUBGRAPHS.md) +- [INVALID_CHAT_HISTORY](../troubleshooting/errors/INVALID_CHAT_HISTORY.md) diff --git a/docs/docs/troubleshooting/errors/INVALID_CHAT_HISTORY.md b/docs/docs/troubleshooting/errors/INVALID_CHAT_HISTORY.md new file mode 100644 index 000000000..121152ba0 --- /dev/null +++ b/docs/docs/troubleshooting/errors/INVALID_CHAT_HISTORY.md @@ -0,0 +1,30 @@ +# INVALID_CHAT_HISTORY + +This error is raised in the prebuilt [create_react_agent][langgraph.prebuilt.chat_agent_executor.create_react_agent] when the `call_model` graph node receives a malformed list of messages. Specifically, it is malformed when there are `AIMessages` with `tool_calls` (LLM requesting to call a tool) that do not have a corresponding `ToolMessage` (result of a tool invocation to return to the LLM). + +There could be a few reasons you're seeing this error: + +1. You manually passed a malformed list of messages when invoking the graph, e.g. `graph.invoke({'messages': [AIMessage(..., tool_calls=[...])]})` +2. The graph was interrupted before receiving updates from the `tools` node (i.e. a list of ToolMessages) +and you invoked it with a an input that is not None or a ToolMessage, +e.g. `graph.invoke({'messages': [HumanMessage(...)]}, config)`. + This interrupt could have been triggered in one of the following ways: + - You manually set `interrupt_before = ['tools']` in `create_react_agent` + - One of the tools raised an error that wasn't handled by the [ToolNode][langgraph.prebuilt.tool_node.ToolNode] (`"tools"`) + +## Troubleshooting + +To resolve this, you can do one of the following: + +1. Don't invoke the graph with a malformed list of messages +2. In case of an interrupt (manual or due to an error) you can: + + - provide ToolMessages that match existing tool calls and call `graph.invoke({'messages': [ToolMessage(...)]})`. + **NOTE**: this will append the messages to the history and run the graph from the START node. + - manually update the state and resume the graph from the interrupt: + + 1. get the list of most recent messages from the graph state with `graph.get_state(config)` + 2. modify the list of messages to either remove unanswered tool calls from AIMessages +or add ToolMessages with tool_call_ids that match unanswered tool calls + 3. call `graph.update_state(config, {'messages': ...})` with the modified list of messages + 4. resume the graph, e.g. call `graph.invoke(None, config)` diff --git a/docs/docs/troubleshooting/errors/index.md b/docs/docs/troubleshooting/errors/index.md index 9a0baab23..c8a21d5d5 100644 --- a/docs/docs/troubleshooting/errors/index.md +++ b/docs/docs/troubleshooting/errors/index.md @@ -7,3 +7,4 @@ Errors referenced below will have an `lc_error_code` property corresponding to o - [INVALID_CONCURRENT_GRAPH_UPDATE](./INVALID_CONCURRENT_GRAPH_UPDATE.md) - [INVALID_GRAPH_NODE_RETURN_VALUE](./INVALID_GRAPH_NODE_RETURN_VALUE.md) - [MULTIPLE_SUBGRAPHS](./MULTIPLE_SUBGRAPHS.md) +- [INVALID_CHAT_HISTORY](./INVALID_CHAT_HISTORY.md) diff --git a/libs/langgraph/langgraph/errors.py b/libs/langgraph/langgraph/errors.py index 114232514..2e3d13120 100644 --- a/libs/langgraph/langgraph/errors.py +++ b/libs/langgraph/langgraph/errors.py @@ -12,6 +12,7 @@ class ErrorCode(Enum): INVALID_CONCURRENT_GRAPH_UPDATE = "INVALID_CONCURRENT_GRAPH_UPDATE" INVALID_GRAPH_NODE_RETURN_VALUE = "INVALID_GRAPH_NODE_RETURN_VALUE" MULTIPLE_SUBGRAPHS = "MULTIPLE_SUBGRAPHS" + INVALID_CHAT_HISTORY = "INVALID_CHAT_HISTORY" def create_error_message(*, message: str, error_code: ErrorCode) -> str: diff --git a/libs/langgraph/langgraph/prebuilt/chat_agent_executor.py b/libs/langgraph/langgraph/prebuilt/chat_agent_executor.py index 5f739c0a6..fc812ccbc 100644 --- a/libs/langgraph/langgraph/prebuilt/chat_agent_executor.py +++ b/libs/langgraph/langgraph/prebuilt/chat_agent_executor.py @@ -11,6 +11,7 @@ from typing_extensions import Annotated, TypedDict from langgraph._api.deprecation import deprecated_parameter +from langgraph.errors import ErrorCode, create_error_message from langgraph.graph import StateGraph from langgraph.graph.graph import CompiledGraph from langgraph.graph.message import add_messages @@ -161,6 +162,37 @@ def _should_bind_tools(model: LanguageModelLike, tools: Sequence[BaseTool]) -> b return False +def _validate_chat_history( + messages: Sequence[BaseMessage], +) -> None: + """Validate that all tool calls in AIMessages have a corresponding ToolMessage.""" + all_tool_calls = [ + tool_call + for message in messages + if isinstance(message, AIMessage) + for tool_call in message.tool_calls + ] + tool_call_ids_with_results = { + message.tool_call_id for message in messages if isinstance(message, ToolMessage) + } + tool_calls_without_results = [ + tool_call + for tool_call in all_tool_calls + if tool_call["id"] not in tool_call_ids_with_results + ] + if not tool_calls_without_results: + return + + error_message = create_error_message( + message="Found AIMessages with tool_calls that do not have a corresponding ToolMessage. " + f"Here are the first few of those tool calls: {tool_calls_without_results[:3]}.\n\n" + "Every tool call (LLM requesting to call a tool) in the message history MUST have a corresponding ToolMessage " + "(result of a tool invocation to return to the LLM) - this is required by most LLM providers.", + error_code=ErrorCode.INVALID_CHAT_HISTORY, + ) + raise ValueError(error_message) + + @deprecated_parameter("messages_modifier", "0.1.9", "state_modifier", removal="0.3.0") def create_react_agent( model: LanguageModelLike, @@ -530,6 +562,7 @@ def should_continue(state: AgentState) -> Literal["tools", "__end__"]: # Define the function that calls the model def call_model(state: AgentState, config: RunnableConfig) -> AgentState: + _validate_chat_history(state["messages"]) response = model_runnable.invoke(state, config) has_tool_calls = isinstance(response, AIMessage) and response.tool_calls all_tools_return_direct = ( @@ -566,6 +599,7 @@ def call_model(state: AgentState, config: RunnableConfig) -> AgentState: return {"messages": [response]} async def acall_model(state: AgentState, config: RunnableConfig) -> AgentState: + _validate_chat_history(state["messages"]) response = await model_runnable.ainvoke(state, config) has_tool_calls = isinstance(response, AIMessage) and response.tool_calls all_tools_return_direct = ( diff --git a/libs/langgraph/tests/test_prebuilt.py b/libs/langgraph/tests/test_prebuilt.py index c77a2cac4..1ea506381 100644 --- a/libs/langgraph/tests/test_prebuilt.py +++ b/libs/langgraph/tests/test_prebuilt.py @@ -46,6 +46,7 @@ create_react_agent, tools_condition, ) +from langgraph.prebuilt.chat_agent_executor import _validate_chat_history from langgraph.prebuilt.tool_node import ( TOOL_CALL_ERROR_TEMPLATE, InjectedState, @@ -378,6 +379,71 @@ def tool2(some_val: int) -> str: create_react_agent(model.bind_tools([tool1]), [tool2]) +def test__validate_messages(): + # empty input + _validate_chat_history([]) + + # single human message + _validate_chat_history( + [ + HumanMessage(content="What's the weather?"), + ] + ) + + # human + AI + _validate_chat_history( + [ + HumanMessage(content="What's the weather?"), + AIMessage(content="The weather is sunny and 75°F."), + ] + ) + + # Answered tool calls + _validate_chat_history( + [ + HumanMessage(content="What's the weather?"), + AIMessage( + content="Let me check that for you.", + tool_calls=[{"id": "call1", "name": "get_weather", "args": {}}], + ), + ToolMessage(content="Sunny, 75°F", tool_call_id="call1"), + AIMessage(content="The weather is sunny and 75°F."), + ] + ) + + # Unanswered tool calls + with pytest.raises(ValueError): + _validate_chat_history( + [ + AIMessage( + content="I'll check that for you.", + tool_calls=[ + {"id": "call1", "name": "get_weather", "args": {}}, + {"id": "call2", "name": "get_time", "args": {}}, + ], + ) + ] + ) + + with pytest.raises(ValueError): + _validate_chat_history( + [ + HumanMessage(content="What's the weather and time?"), + AIMessage( + content="I'll check that for you.", + tool_calls=[ + {"id": "call1", "name": "get_weather", "args": {}}, + {"id": "call2", "name": "get_time", "args": {}}, + ], + ), + ToolMessage(content="Sunny, 75°F", tool_call_id="call1"), + AIMessage( + content="The weather is sunny and 75°F. Let me check the time." + ), + ] + ) + + def test__infer_handled_types() -> None: def handle(e): # type: ignore return ""