diff --git a/core/cat/factory/custom_llm.py b/core/cat/factory/custom_llm.py index 092ecebe..3e77f482 100644 --- a/core/cat/factory/custom_llm.py +++ b/core/cat/factory/custom_llm.py @@ -10,13 +10,12 @@ def _llm_type(self): def _call(self, prompt, stop=None): return "AI: You did not configure a Language Model. " \ - "Do it in the settings!" + "Do it in the settings!" # elaborated from # https://python.langchain.com/en/latest/modules/models/llms/examples/custom_llm.html class LLMCustom(LLM): - # endpoint where custom LLM service accepts requests url: str @@ -31,11 +30,11 @@ def _llm_type(self) -> str: return "custom" def _call( - self, - prompt: str, - stop: Optional[List[str]] = None, - # run_manager: Optional[CallbackManagerForLLMRun] = None, - run_manager: Optional[Any] = None, + self, + prompt: str, + stop: Optional[List[str]] = None, + # run_manager: Optional[CallbackManagerForLLMRun] = None, + run_manager: Optional[Any] = None, ) -> str: request_body = { @@ -46,13 +45,13 @@ def _call( try: response_json = requests.post(self.url, json=request_body).json() - except Exception: - raise Exception("Custom LLM endpoint error " - "during http POST request") + except Exception as exc: + raise ValueError("Custom LLM endpoint error " + "during http POST request") from exc generated_text = response_json["text"] - return f"AI: {generated_text}" + return generated_text @property def _identifying_params(self) -> Mapping[str, Any]: diff --git a/core/cat/factory/llm.py b/core/cat/factory/llm.py index 7cfd70dd..92a1d715 100644 --- a/core/cat/factory/llm.py +++ b/core/cat/factory/llm.py @@ -36,7 +36,6 @@ class Config: class LLMCustomConfig(LLMSettings): - url: str auth_key: str = "optional_auth_key" options: str = "{}" @@ -45,16 +44,20 @@ class LLMCustomConfig(LLMSettings): # instantiate Custom LLM from configuration @classmethod def get_llm_from_config(cls, config): + options = config["options"] # options are inserted as a string in the admin - if type(config["options"]) == str: - config["options"] = json.loads(config["options"]) + if isinstance(options, str): + if options != "": + config["options"] = json.loads(options) + else: + config["options"] = {} return cls._pyclass(**config) class Config: schema_extra = { "humanReadableName": "Custom LLM", - "description": + "description": "LLM on a custom endpoint. " "See docs for examples.", } @@ -80,7 +83,7 @@ class LLMOpenAIConfig(LLMSettings): class Config: schema_extra = { "humanReadableName": "OpenAI GPT-3", - "description": + "description": "OpenAI GPT-3. More expensive but " "also more flexible than ChatGPT.", } @@ -138,6 +141,7 @@ class Config: "description": "Configuration for Cohere language model", } + # https://python.langchain.com/en/latest/modules/models/llms/integrations/huggingface_textgen_inference.html class LLMHuggingFaceTextGenInferenceConfig(LLMSettings): inference_server_url: str @@ -155,6 +159,7 @@ class Config: "description": "Configuration for HuggingFace TextGen Inference", } + class LLMHuggingFaceHubConfig(LLMSettings): # model_kwargs = { # "generation_config": { diff --git a/core/cat/looking_glass/agent_manager.py b/core/cat/looking_glass/agent_manager.py index c2f2a8ec..b4cb77cb 100644 --- a/core/cat/looking_glass/agent_manager.py +++ b/core/cat/looking_glass/agent_manager.py @@ -1,11 +1,6 @@ -import re -import traceback -import json -from copy import copy - from langchain.prompts import PromptTemplate from langchain.chains import LLMChain -from langchain.agents import AgentExecutor, LLMSingleActionAgent, AgentOutputParser +from langchain.agents import AgentExecutor, LLMSingleActionAgent from cat.looking_glass.prompts import ToolPromptTemplate from cat.looking_glass.output_parser import ToolOutputParser @@ -33,7 +28,7 @@ def execute_tool_agent(self, agent_input, allowed_tools): allowed_tools_names = [t.name for t in allowed_tools] prompt = ToolPromptTemplate( - #template= TODO: get from hook, + template = self.cat.mad_hatter.execute_hook("agent_prompt_instructions"), tools=allowed_tools, # This omits the `agent_scratchpad`, `tools`, and `tool_names` variables because those are generated dynamically # This includes the `intermediate_steps` variable because it is needed to fill the scratchpad @@ -109,62 +104,59 @@ def execute_agent(self, agent_input): return fast_reply prompt_prefix = mad_hatter.execute_hook("agent_prompt_prefix") - #prompt_format_instructions = mad_hatter.execute_hook("agent_prompt_instructions") prompt_suffix = mad_hatter.execute_hook("agent_prompt_suffix") - #input_variables = [ - # "input", - # "chat_history", - # "episodic_memory", - # "declarative_memory", - # "agent_scratchpad", - #] - - #input_variables = mad_hatter.execute_hook("before_agent_creates_prompt", input_variables, - # " ".join([prompt_prefix, prompt_format_instructions, prompt_suffix])) - + allowed_tools = mad_hatter.execute_hook("agent_allowed_tools") # Try to get information from tools if there is some allowed - allowed_tools = mad_hatter.execute_hook("agent_allowed_tools") - tools_result = None if len(allowed_tools) > 0: + + log(f"{len(allowed_tools)} allowed tools retrived.", "DEBUG") + try: tools_result = self.execute_tool_agent(agent_input, allowed_tools) + + # If tools_result["output"] is None the LLM has used the fake tool none_of_the_others + # so no relevant information has been obtained from the tools. + if tools_result["output"] != None: + + # Extract of intermediate steps in the format ((tool_name, tool_input), output) + used_tools = list(map(lambda x:((x[0].tool, x[0].tool_input), x[1]), tools_result["intermediate_steps"])) + + # Get the name of the tools that have return_direct + return_direct_tools = [] + for t in allowed_tools: + if t.return_direct: + return_direct_tools.append(t.name) + + # execute_tool_agent returns immediately when a tool with return_direct is called, + # so if one is used it is definitely the last one used + if used_tools[-1][0][0] in return_direct_tools: + # intermediate_steps still contains the information of all the tools used even if their output is not returned + tools_result["intermediate_steps"] = used_tools + return tools_result + + #Adding the tools_output key in agent input, needed by the memory chain + agent_input["tools_output"] = "## Tools output: \n" + tools_result["output"] if tools_result["output"] else "" + + # Execute the memory chain + out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) + + # If some tools are used the intermediate step are added to the agent output + out["intermediate_steps"] = used_tools + + #Early return + return out + except Exception as e: error_description = str(e) log(error_description, "ERROR") + #If an exeption occur in the execute_tool_agent or there is no allowed tools execute only the memory chain + #Adding the tools_output key in agent input, needed by the memory chain - if tools_result != None: - - # Extract of intermediate steps in the format ((tool_name, tool_input), output) - used_tools = list(map(lambda x:((x[0].tool, x[0].tool_input), x[1]), tools_result["intermediate_steps"])) - - # Get the name of the tools that have return_direct - return_direct_tools = [] - for t in allowed_tools: - if t.return_direct: - return_direct_tools.append(t.name) - - # execute_tool_agent returns immediately when a tool with return_direct is called, - # so if one is used it is definitely the last one used - if used_tools[-1][0][0] in return_direct_tools: - # intermediate_steps still contains the information of all the tools used even if their output is not returned - tools_result["intermediate_steps"] = used_tools - return tools_result - - # If tools_result["output"] is None the LLM has used the fake tool none_of_the_others - # so no relevant information has been obtained from the tools. - agent_input["tools_output"] = "## Tools output: \n" + tools_result["output"] if tools_result["output"] else "" - - # Execute the memory chain - out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) - - # If some tools are used the intermediate step are added to the agent output - out["intermediate_steps"] = used_tools - else: - agent_input["tools_output"] = "" - # Execute the memory chain - out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) + agent_input["tools_output"] = "" + # Execute the memory chain + out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) return out diff --git a/core/cat/looking_glass/cheshire_cat.py b/core/cat/looking_glass/cheshire_cat.py index cf1be6ea..b9f82a1f 100644 --- a/core/cat/looking_glass/cheshire_cat.py +++ b/core/cat/looking_glass/cheshire_cat.py @@ -376,7 +376,7 @@ def __call__(self, user_message_json): # We grab the LLM output here anyway, so small and # non instruction-fine-tuned models can still be used. error_description = str(e) - log("LLM does not respect prompt instructions", "ERROR") + log(error_description, "ERROR") if not "Could not parse LLM output: `" in error_description: raise e diff --git a/core/cat/looking_glass/output_parser.py b/core/cat/looking_glass/output_parser.py index 23874b41..b43cff5a 100644 --- a/core/cat/looking_glass/output_parser.py +++ b/core/cat/looking_glass/output_parser.py @@ -1,7 +1,7 @@ import re from langchain.agents import AgentOutputParser from langchain.schema import AgentAction, AgentFinish, OutputParserException -from typing import List, Union +from typing import Union class ToolOutputParser(AgentOutputParser): @@ -23,8 +23,11 @@ def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]: if not match: raise OutputParserException(f"Could not parse LLM output: `{llm_output}`") - # Check if agent decidet not tool is usefull - if "none_of_the_others" in llm_output: + # Extract action + action = match.group(1).strip() + action_input = match.group(2) + + if action == "none_of_the_others": return AgentFinish( # Return values is generally always a dictionary with a single `output` key # It is not recommended to try anything else at the moment :) @@ -32,8 +35,5 @@ def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]: log=llm_output, ) - # Extract action - action = match.group(1).strip() - action_input = match.group(2) # Return the action and action input return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output) \ No newline at end of file diff --git a/core/cat/looking_glass/prompts.py b/core/cat/looking_glass/prompts.py index 333232a4..44b99406 100644 --- a/core/cat/looking_glass/prompts.py +++ b/core/cat/looking_glass/prompts.py @@ -4,34 +4,9 @@ from langchain.agents.tools import BaseTool from langchain.prompts import StringPromptTemplate -# TODO: get by hook -DEFAULT_TOOL_TEMPLATE = """Answer the following question: `{input}` -You can only reply using these tools: - -{tools} -none_of_the_others: none_of_the_others(None) - Use this tool if none of the others tools help. Input is always None. - -If you want to use tools, use the following format: -Action: the name of the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... -Action: the name of the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action - -When you have a final answer respond with: -Final Answer: the final answer to the original input question - -Begin! - -Question: {input} -{agent_scratchpad}""" - - class ToolPromptTemplate(StringPromptTemplate): # The template to use - template: str = DEFAULT_TOOL_TEMPLATE + template: str # The list of tools available tools: List[BaseTool] diff --git a/core/cat/mad_hatter/core_plugin/hooks/prompt.py b/core/cat/mad_hatter/core_plugin/hooks/prompt.py index 3f288aac..2ebc3012 100644 --- a/core/cat/mad_hatter/core_plugin/hooks/prompt.py +++ b/core/cat/mad_hatter/core_plugin/hooks/prompt.py @@ -8,7 +8,6 @@ from typing import List, Dict from datetime import timedelta from langchain.docstore.document import Document -from langchain.agents.conversational import prompt from cat.utils import verbal_timedelta from cat.mad_hatter.decorators import hook @@ -87,13 +86,32 @@ def agent_prompt_instructions(cat) -> str: """ - # Check if procedural memory is disabled - prompt_settings = cat.working_memory["user_message_json"]["prompt_settings"] - if not prompt_settings["use_procedural_memory"]: - return "" + DEFAULT_TOOL_TEMPLATE = """Answer the following question: `{input}` + You can only reply using these tools: + + {tools} + none_of_the_others: none_of_the_others(None) - Use this tool if none of the others tools help. Input is always None. + + If you want to use tools, use the following format: + Action: the name of the action to take, should be one of [{tool_names}] + Action Input: the input to the action + Observation: the result of the action + ... + Action: the name of the action to take, should be one of [{tool_names}] + Action Input: the input to the action + Observation: the result of the action + + When you have a final answer respond with: + Final Answer: the final answer to the original input question + + Begin! + + Question: {input} + {agent_scratchpad}""" + # here we piggy back directly on langchain agent instructions. Different instructions will require a different OutputParser - return prompt.FORMAT_INSTRUCTIONS + return DEFAULT_TOOL_TEMPLATE @hook(priority=0) diff --git a/core/pyproject.toml b/core/pyproject.toml index cdceec40..7d77edc5 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "Cheshire-Cat" description = "Open source and customizable AI architecture" -version = "1.0.1" +version = "1.0.2" requires-python = ">=3.10" license = { file="LICENSE" } authors = [