From 89125b21d31e986287f69ac5bf8b101af7c633c1 Mon Sep 17 00:00:00 2001 From: Nicola Date: Thu, 17 Aug 2023 18:07:42 +0200 Subject: [PATCH 01/11] fix bug in `get_llm_from_config` for `LLMCustomConfig` --- core/cat/factory/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cat/factory/llm.py b/core/cat/factory/llm.py index 7cfd70dd..6a0b2564 100644 --- a/core/cat/factory/llm.py +++ b/core/cat/factory/llm.py @@ -46,7 +46,7 @@ class LLMCustomConfig(LLMSettings): @classmethod def get_llm_from_config(cls, config): # options are inserted as a string in the admin - if type(config["options"]) == str: + if isinstance(config["options"], dict): config["options"] = json.loads(config["options"]) return cls._pyclass(**config) From 934584d0fe2ec43fda992d9639d6f1084b3a6b57 Mon Sep 17 00:00:00 2001 From: Nicola Date: Fri, 18 Aug 2023 10:11:38 +0200 Subject: [PATCH 02/11] add header to `request.post` --- core/cat/factory/custom_llm.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/core/cat/factory/custom_llm.py b/core/cat/factory/custom_llm.py index 092ecebe..d3b07c6d 100644 --- a/core/cat/factory/custom_llm.py +++ b/core/cat/factory/custom_llm.py @@ -1,6 +1,7 @@ from typing import Optional, List, Any, Mapping, Dict import requests from langchain.llms.base import LLM +from cat.log import log class LLMDefault(LLM): @@ -10,13 +11,12 @@ def _llm_type(self): def _call(self, prompt, stop=None): return "AI: You did not configure a Language Model. " \ - "Do it in the settings!" + "Do it in the settings!" # elaborated from # https://python.langchain.com/en/latest/modules/models/llms/examples/custom_llm.html class LLMCustom(LLM): - # endpoint where custom LLM service accepts requests url: str @@ -31,11 +31,11 @@ def _llm_type(self) -> str: return "custom" def _call( - self, - prompt: str, - stop: Optional[List[str]] = None, - # run_manager: Optional[CallbackManagerForLLMRun] = None, - run_manager: Optional[Any] = None, + self, + prompt: str, + stop: Optional[List[str]] = None, + # run_manager: Optional[CallbackManagerForLLMRun] = None, + run_manager: Optional[Any] = None, ) -> str: request_body = { @@ -44,11 +44,16 @@ def _call( "options": self.options } + headers = { + 'accept': 'application/json', + 'Content-Type': 'application/json' + } + try: - response_json = requests.post(self.url, json=request_body).json() - except Exception: - raise Exception("Custom LLM endpoint error " - "during http POST request") + response_json = requests.post(self.url, json=request_body, headers=headers).json() + except Exception as exc: + raise ValueError("Custom LLM endpoint error " + "during http POST request") from exc generated_text = response_json["text"] From 2577acf6ddbd7f49437aca86d026ec32e13318ed Mon Sep 17 00:00:00 2001 From: Nicola Date: Fri, 18 Aug 2023 12:18:15 +0200 Subject: [PATCH 03/11] restored type check to string --- core/cat/factory/custom_llm.py | 2 +- core/cat/factory/llm.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/core/cat/factory/custom_llm.py b/core/cat/factory/custom_llm.py index d3b07c6d..73b99d1a 100644 --- a/core/cat/factory/custom_llm.py +++ b/core/cat/factory/custom_llm.py @@ -50,7 +50,7 @@ def _call( } try: - response_json = requests.post(self.url, json=request_body, headers=headers).json() + response_json = requests.post(self.url, json=request_body).json() except Exception as exc: raise ValueError("Custom LLM endpoint error " "during http POST request") from exc diff --git a/core/cat/factory/llm.py b/core/cat/factory/llm.py index 6a0b2564..92a1d715 100644 --- a/core/cat/factory/llm.py +++ b/core/cat/factory/llm.py @@ -36,7 +36,6 @@ class Config: class LLMCustomConfig(LLMSettings): - url: str auth_key: str = "optional_auth_key" options: str = "{}" @@ -45,16 +44,20 @@ class LLMCustomConfig(LLMSettings): # instantiate Custom LLM from configuration @classmethod def get_llm_from_config(cls, config): + options = config["options"] # options are inserted as a string in the admin - if isinstance(config["options"], dict): - config["options"] = json.loads(config["options"]) + if isinstance(options, str): + if options != "": + config["options"] = json.loads(options) + else: + config["options"] = {} return cls._pyclass(**config) class Config: schema_extra = { "humanReadableName": "Custom LLM", - "description": + "description": "LLM on a custom endpoint. " "See docs for examples.", } @@ -80,7 +83,7 @@ class LLMOpenAIConfig(LLMSettings): class Config: schema_extra = { "humanReadableName": "OpenAI GPT-3", - "description": + "description": "OpenAI GPT-3. More expensive but " "also more flexible than ChatGPT.", } @@ -138,6 +141,7 @@ class Config: "description": "Configuration for Cohere language model", } + # https://python.langchain.com/en/latest/modules/models/llms/integrations/huggingface_textgen_inference.html class LLMHuggingFaceTextGenInferenceConfig(LLMSettings): inference_server_url: str @@ -155,6 +159,7 @@ class Config: "description": "Configuration for HuggingFace TextGen Inference", } + class LLMHuggingFaceHubConfig(LLMSettings): # model_kwargs = { # "generation_config": { From 93ddc6e0c294cdc6d7f1d66c5ba28669438273dc Mon Sep 17 00:00:00 2001 From: Nicola Date: Fri, 18 Aug 2023 16:56:09 +0200 Subject: [PATCH 04/11] removed unnecessary headers --- core/cat/factory/custom_llm.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/core/cat/factory/custom_llm.py b/core/cat/factory/custom_llm.py index 73b99d1a..3e77f482 100644 --- a/core/cat/factory/custom_llm.py +++ b/core/cat/factory/custom_llm.py @@ -1,7 +1,6 @@ from typing import Optional, List, Any, Mapping, Dict import requests from langchain.llms.base import LLM -from cat.log import log class LLMDefault(LLM): @@ -44,11 +43,6 @@ def _call( "options": self.options } - headers = { - 'accept': 'application/json', - 'Content-Type': 'application/json' - } - try: response_json = requests.post(self.url, json=request_body).json() except Exception as exc: @@ -57,7 +51,7 @@ def _call( generated_text = response_json["text"] - return f"AI: {generated_text}" + return generated_text @property def _identifying_params(self) -> Mapping[str, Any]: From 7ae5bd0e7bada6deaa6a03f7ba00d769469d0f9e Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:46:37 +0200 Subject: [PATCH 05/11] First extract the used tool and then check if is it none_of_the_others --- core/cat/looking_glass/output_parser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/cat/looking_glass/output_parser.py b/core/cat/looking_glass/output_parser.py index 23874b41..451de544 100644 --- a/core/cat/looking_glass/output_parser.py +++ b/core/cat/looking_glass/output_parser.py @@ -23,8 +23,11 @@ def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]: if not match: raise OutputParserException(f"Could not parse LLM output: `{llm_output}`") - # Check if agent decidet not tool is usefull - if "none_of_the_others" in llm_output: + # Extract action + action = match.group(1).strip() + action_input = match.group(2) + + if action == "none_of_the_others": return AgentFinish( # Return values is generally always a dictionary with a single `output` key # It is not recommended to try anything else at the moment :) @@ -32,8 +35,5 @@ def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]: log=llm_output, ) - # Extract action - action = match.group(1).strip() - action_input = match.group(2) # Return the action and action input return AgentAction(tool=action, tool_input=action_input.strip(" ").strip('"'), log=llm_output) \ No newline at end of file From cf34afcc8f9afdb15aaaac4a54e929174e11b1bb Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:52:58 +0200 Subject: [PATCH 06/11] Small refactoring and corrected the use of none_of_the_others tool If the `none_of_the_others` tool is used, `execute_tool_agent` returns the dictionary `{"output": None}` so the check on line 139 must be `tools_output["output"] != None` and not `tools_output != None ` --- core/cat/looking_glass/agent_manager.py | 76 ++++++++++++++----------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/core/cat/looking_glass/agent_manager.py b/core/cat/looking_glass/agent_manager.py index c2f2a8ec..af900cdc 100644 --- a/core/cat/looking_glass/agent_manager.py +++ b/core/cat/looking_glass/agent_manager.py @@ -124,47 +124,57 @@ def execute_agent(self, agent_input): # " ".join([prompt_prefix, prompt_format_instructions, prompt_suffix])) - # Try to get information from tools if there is some allowed allowed_tools = mad_hatter.execute_hook("agent_allowed_tools") - tools_result = None + + # Try to get information from tools if there is some allowed if len(allowed_tools) > 0: + + log(f"{len(allowed_tools)} allowed tools retrived.", "DEBUG") + try: tools_result = self.execute_tool_agent(agent_input, allowed_tools) + + # If tools_result["output"] is None the LLM has used the fake tool none_of_the_others + # so no relevant information has been obtained from the tools. + if tools_result["output"] != None: + + # Extract of intermediate steps in the format ((tool_name, tool_input), output) + used_tools = list(map(lambda x:((x[0].tool, x[0].tool_input), x[1]), tools_result["intermediate_steps"])) + + # Get the name of the tools that have return_direct + return_direct_tools = [] + for t in allowed_tools: + if t.return_direct: + return_direct_tools.append(t.name) + + # execute_tool_agent returns immediately when a tool with return_direct is called, + # so if one is used it is definitely the last one used + if used_tools[-1][0][0] in return_direct_tools: + # intermediate_steps still contains the information of all the tools used even if their output is not returned + tools_result["intermediate_steps"] = used_tools + return tools_result + + #Adding the tools_output key in agent input, needed by the memory chain + agent_input["tools_output"] = "## Tools output: \n" + tools_result["output"] if tools_result["output"] else "" + + # Execute the memory chain + out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) + + # If some tools are used the intermediate step are added to the agent output + out["intermediate_steps"] = used_tools + + #Early return + return out + except Exception as e: error_description = str(e) log(error_description, "ERROR") + #If an exeption occur in the execute_tool_agent or there is no allowed tools execute only the memory chain + #Adding the tools_output key in agent input, needed by the memory chain - if tools_result != None: - - # Extract of intermediate steps in the format ((tool_name, tool_input), output) - used_tools = list(map(lambda x:((x[0].tool, x[0].tool_input), x[1]), tools_result["intermediate_steps"])) - - # Get the name of the tools that have return_direct - return_direct_tools = [] - for t in allowed_tools: - if t.return_direct: - return_direct_tools.append(t.name) - - # execute_tool_agent returns immediately when a tool with return_direct is called, - # so if one is used it is definitely the last one used - if used_tools[-1][0][0] in return_direct_tools: - # intermediate_steps still contains the information of all the tools used even if their output is not returned - tools_result["intermediate_steps"] = used_tools - return tools_result - - # If tools_result["output"] is None the LLM has used the fake tool none_of_the_others - # so no relevant information has been obtained from the tools. - agent_input["tools_output"] = "## Tools output: \n" + tools_result["output"] if tools_result["output"] else "" - - # Execute the memory chain - out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) - - # If some tools are used the intermediate step are added to the agent output - out["intermediate_steps"] = used_tools - else: - agent_input["tools_output"] = "" - # Execute the memory chain - out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) + agent_input["tools_output"] = "" + # Execute the memory chain + out = self.execute_memory_chain(agent_input, prompt_prefix, prompt_suffix) return out From 2055709635459aa1f29e25bda241e6bdf02ae93a Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:57:52 +0200 Subject: [PATCH 07/11] Removed unnecessary log --- core/cat/looking_glass/cheshire_cat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cat/looking_glass/cheshire_cat.py b/core/cat/looking_glass/cheshire_cat.py index cf1be6ea..b9f82a1f 100644 --- a/core/cat/looking_glass/cheshire_cat.py +++ b/core/cat/looking_glass/cheshire_cat.py @@ -376,7 +376,7 @@ def __call__(self, user_message_json): # We grab the LLM output here anyway, so small and # non instruction-fine-tuned models can still be used. error_description = str(e) - log("LLM does not respect prompt instructions", "ERROR") + log(error_description, "ERROR") if not "Could not parse LLM output: `" in error_description: raise e From 85957a4ea514d289f47238e1b584d97120e88bea Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:12:11 +0200 Subject: [PATCH 08/11] Moved `execute_tool_agent` prompt in the hook `agent_prompt_instructions` --- core/cat/looking_glass/prompts.py | 27 +---------------- .../mad_hatter/core_plugin/hooks/prompt.py | 29 +++++++++++++++---- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/core/cat/looking_glass/prompts.py b/core/cat/looking_glass/prompts.py index 333232a4..44b99406 100644 --- a/core/cat/looking_glass/prompts.py +++ b/core/cat/looking_glass/prompts.py @@ -4,34 +4,9 @@ from langchain.agents.tools import BaseTool from langchain.prompts import StringPromptTemplate -# TODO: get by hook -DEFAULT_TOOL_TEMPLATE = """Answer the following question: `{input}` -You can only reply using these tools: - -{tools} -none_of_the_others: none_of_the_others(None) - Use this tool if none of the others tools help. Input is always None. - -If you want to use tools, use the following format: -Action: the name of the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action -... -Action: the name of the action to take, should be one of [{tool_names}] -Action Input: the input to the action -Observation: the result of the action - -When you have a final answer respond with: -Final Answer: the final answer to the original input question - -Begin! - -Question: {input} -{agent_scratchpad}""" - - class ToolPromptTemplate(StringPromptTemplate): # The template to use - template: str = DEFAULT_TOOL_TEMPLATE + template: str # The list of tools available tools: List[BaseTool] diff --git a/core/cat/mad_hatter/core_plugin/hooks/prompt.py b/core/cat/mad_hatter/core_plugin/hooks/prompt.py index 3f288aac..943ebbcd 100644 --- a/core/cat/mad_hatter/core_plugin/hooks/prompt.py +++ b/core/cat/mad_hatter/core_plugin/hooks/prompt.py @@ -87,13 +87,32 @@ def agent_prompt_instructions(cat) -> str: """ - # Check if procedural memory is disabled - prompt_settings = cat.working_memory["user_message_json"]["prompt_settings"] - if not prompt_settings["use_procedural_memory"]: - return "" + DEFAULT_TOOL_TEMPLATE = """Answer the following question: `{input}` + You can only reply using these tools: + + {tools} + none_of_the_others: none_of_the_others(None) - Use this tool if none of the others tools help. Input is always None. + + If you want to use tools, use the following format: + Action: the name of the action to take, should be one of [{tool_names}] + Action Input: the input to the action + Observation: the result of the action + ... + Action: the name of the action to take, should be one of [{tool_names}] + Action Input: the input to the action + Observation: the result of the action + + When you have a final answer respond with: + Final Answer: the final answer to the original input question + + Begin! + + Question: {input} + {agent_scratchpad}""" + # here we piggy back directly on langchain agent instructions. Different instructions will require a different OutputParser - return prompt.FORMAT_INSTRUCTIONS + return DEFAULT_TOOL_TEMPLATE @hook(priority=0) From 26181f5b03bd33597b6b738e3376d6ac209b0b0d Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:12:45 +0200 Subject: [PATCH 09/11] Get `execute_tool_agent` prompt by hook --- core/cat/looking_glass/agent_manager.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/core/cat/looking_glass/agent_manager.py b/core/cat/looking_glass/agent_manager.py index c2f2a8ec..a2643628 100644 --- a/core/cat/looking_glass/agent_manager.py +++ b/core/cat/looking_glass/agent_manager.py @@ -33,7 +33,7 @@ def execute_tool_agent(self, agent_input, allowed_tools): allowed_tools_names = [t.name for t in allowed_tools] prompt = ToolPromptTemplate( - #template= TODO: get from hook, + template = self.cat.mad_hatter.execute_hook("agent_prompt_instructions"), tools=allowed_tools, # This omits the `agent_scratchpad`, `tools`, and `tool_names` variables because those are generated dynamically # This includes the `intermediate_steps` variable because it is needed to fill the scratchpad @@ -109,21 +109,8 @@ def execute_agent(self, agent_input): return fast_reply prompt_prefix = mad_hatter.execute_hook("agent_prompt_prefix") - #prompt_format_instructions = mad_hatter.execute_hook("agent_prompt_instructions") prompt_suffix = mad_hatter.execute_hook("agent_prompt_suffix") - #input_variables = [ - # "input", - # "chat_history", - # "episodic_memory", - # "declarative_memory", - # "agent_scratchpad", - #] - - #input_variables = mad_hatter.execute_hook("before_agent_creates_prompt", input_variables, - # " ".join([prompt_prefix, prompt_format_instructions, prompt_suffix])) - - # Try to get information from tools if there is some allowed allowed_tools = mad_hatter.execute_hook("agent_allowed_tools") tools_result = None From 4f202fe157bb626804487198717c35261406f37b Mon Sep 17 00:00:00 2001 From: Emanuele Morrone <67059270+Pingdred@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:37:53 +0200 Subject: [PATCH 10/11] Removed unused import --- core/cat/looking_glass/agent_manager.py | 7 +------ core/cat/looking_glass/output_parser.py | 2 +- core/cat/mad_hatter/core_plugin/hooks/prompt.py | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/core/cat/looking_glass/agent_manager.py b/core/cat/looking_glass/agent_manager.py index a2643628..8ca67390 100644 --- a/core/cat/looking_glass/agent_manager.py +++ b/core/cat/looking_glass/agent_manager.py @@ -1,11 +1,6 @@ -import re -import traceback -import json -from copy import copy - from langchain.prompts import PromptTemplate from langchain.chains import LLMChain -from langchain.agents import AgentExecutor, LLMSingleActionAgent, AgentOutputParser +from langchain.agents import AgentExecutor, LLMSingleActionAgent from cat.looking_glass.prompts import ToolPromptTemplate from cat.looking_glass.output_parser import ToolOutputParser diff --git a/core/cat/looking_glass/output_parser.py b/core/cat/looking_glass/output_parser.py index 23874b41..0c07b701 100644 --- a/core/cat/looking_glass/output_parser.py +++ b/core/cat/looking_glass/output_parser.py @@ -1,7 +1,7 @@ import re from langchain.agents import AgentOutputParser from langchain.schema import AgentAction, AgentFinish, OutputParserException -from typing import List, Union +from typing import Union class ToolOutputParser(AgentOutputParser): diff --git a/core/cat/mad_hatter/core_plugin/hooks/prompt.py b/core/cat/mad_hatter/core_plugin/hooks/prompt.py index 943ebbcd..2ebc3012 100644 --- a/core/cat/mad_hatter/core_plugin/hooks/prompt.py +++ b/core/cat/mad_hatter/core_plugin/hooks/prompt.py @@ -8,7 +8,6 @@ from typing import List, Dict from datetime import timedelta from langchain.docstore.document import Document -from langchain.agents.conversational import prompt from cat.utils import verbal_timedelta from cat.mad_hatter.decorators import hook From 109b58b77b2217b48e1202036e9f053334ef32d9 Mon Sep 17 00:00:00 2001 From: Piero Savastano Date: Mon, 21 Aug 2023 13:00:57 +0200 Subject: [PATCH 11/11] version 1.0.2 bug fixes and new hook for tool prompt --- core/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/pyproject.toml b/core/pyproject.toml index cdceec40..7d77edc5 100644 --- a/core/pyproject.toml +++ b/core/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "Cheshire-Cat" description = "Open source and customizable AI architecture" -version = "1.0.1" +version = "1.0.2" requires-python = ">=3.10" license = { file="LICENSE" } authors = [