From 7ae5bd0e7bada6deaa6a03f7ba00d769469d0f9e Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:46:37 +0200 Subject: [PATCH 1/3] First extract the used tool and then check if is it none_of_the_others --- core/cat/looking_glass/output_parser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/cat/looking_glass/output_parser.py b/core/cat/looking_glass/output_parser.py index 23874b41..451de544 100644 --- a/core/cat/looking_glass/output_parser.py +++ b/core/cat/looking_glass/output_parser.py @@ -23,8 +23,11 @@ def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]: if not match: raise OutputParserException(f"Could not parse LLM output: `{llm_output}`") - # Check if agent decidet not tool is usefull - if "none_of_the_others" in llm_output: + # Extract action + action = match.group(1).strip() + action_input = match.group(2) + + if action == "none_of_the_others": return AgentFinish( # Return values is generally always a dictionary with a single `output` key # It is not recommended to try anything else at the moment :) @@ -32,8 +35,5 @@ def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]: log=llm_output, ) - # Extract action - action = match.group(1).strip() - action_input = match.group(2) # Return the action and action input return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output) \ No newline at end of file From cf34afcc8f9afdb15aaaac4a54e929174e11b1bb Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:52:58 +0200 Subject: [PATCH 2/3] Small refactoring and corrected the use of none_of_the_others tool If the `none_of_the_others` tool is used, `execute_tool_agent` returns the dictionary `{"output": None}` so the check on line 139 must be `tools_output["output"] != None` and not `tools_output != None ` --- core/cat/looking_glass/agent_manager.py | 76 ++++++++++++++----------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/core/cat/looking_glass/agent_manager.py b/core/cat/looking_glass/agent_manager.py index c2f2a8ec..af900cdc 100644 --- a/core/cat/looking_glass/agent_manager.py +++ b/core/cat/looking_glass/agent_manager.py @@ -124,47 +124,57 @@ def execute_agent(self, agent_input): # " ".join([prompt_prefix, prompt_format_instructions, prompt_suffix])) - # Try to get information from tools if there is some allowed allowed_tools = mad_hatter.execute_hook("agent_allowed_tools") - tools_result = None + + # Try to get information from tools if there is some allowed if len(allowed_tools) > 0: + + log(f"{len(allowed_tools)} allowed tools retrived.", "DEBUG") + try: tools_result = self.execute_tool_agent(agent_input, allowed_tools) + + # If tools_result["output"] is None the LLM has used the fake tool none_of_the_others + # so no relevant information has been obtained from the tools. + if tools_result["output"] != None: + + # Extract of intermediate steps in the format ((tool_name, tool_input), output) + used_tools = list(map(lambda x:((x[0].tool, x[0].tool_input), x[1]), tools_result["intermediate_steps"])) + + # Get the name of the tools that have return_direct + return_direct_tools = [] + for t in allowed_tools: + if t.return_direct: + return_direct_tools.append(t.name) + + # execute_tool_agent returns immediately when a tool with return_direct is called, + # so if one is used it is definitely the last one used + if used_tools[-1][0][0] in return_direct_tools: + # intermediate_steps still contains the information of all the tools used even if their output is not returned + tools_result["intermediate_steps"] = used_tools + return tools_result + + #Adding the tools_output key in agent input, needed by the memory chain + agent_input["tools_output"] = "## Tools output: \n" + tools_result["output"] if tools_result["output"] else "" + + # Execute the memory chain + out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) + + # If some tools are used the intermediate step are added to the agent output + out["intermediate_steps"] = used_tools + + #Early return + return out + except Exception as e: error_description = str(e) log(error_description, "ERROR") + #If an exeption occur in the execute_tool_agent or there is no allowed tools execute only the memory chain + #Adding the tools_output key in agent input, needed by the memory chain - if tools_result != None: - - # Extract of intermediate steps in the format ((tool_name, tool_input), output) - used_tools = list(map(lambda x:((x[0].tool, x[0].tool_input), x[1]), tools_result["intermediate_steps"])) - - # Get the name of the tools that have return_direct - return_direct_tools = [] - for t in allowed_tools: - if t.return_direct: - return_direct_tools.append(t.name) - - # execute_tool_agent returns immediately when a tool with return_direct is called, - # so if one is used it is definitely the last one used - if used_tools[-1][0][0] in return_direct_tools: - # intermediate_steps still contains the information of all the tools used even if their output is not returned - tools_result["intermediate_steps"] = used_tools - return tools_result - - # If tools_result["output"] is None the LLM has used the fake tool none_of_the_others - # so no relevant information has been obtained from the tools. - agent_input["tools_output"] = "## Tools output: \n" + tools_result["output"] if tools_result["output"] else "" - - # Execute the memory chain - out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) - - # If some tools are used the intermediate step are added to the agent output - out["intermediate_steps"] = used_tools - else: - agent_input["tools_output"] = "" - # Execute the memory chain - out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) + agent_input["tools_output"] = "" + # Execute the memory chain + out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) return out From 2055709635459aa1f29e25bda241e6bdf02ae93a Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:57:52 +0200 Subject: [PATCH 3/3] Removed unnecessary log --- core/cat/looking_glass/cheshire_cat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cat/looking_glass/cheshire_cat.py b/core/cat/looking_glass/cheshire_cat.py index cf1be6ea..b9f82a1f 100644 --- a/core/cat/looking_glass/cheshire_cat.py +++ b/core/cat/looking_glass/cheshire_cat.py @@ -376,7 +376,7 @@ def __call__(self, user_message_json): # We grab the LLM output here anyway, so small and # non instruction-fine-tuned models can still be used. error_description = str(e) - log("LLM does not respect prompt instructions", "ERROR") + log(error_description, "ERROR") if not "Could not parse LLM output: `" in error_description: raise e