Merge branch 'AlboCode-main' into develop

core/cat/factory/llm.py

@@ -97,6 +97,7 @@ class LLMOpenAIChatConfig(LLMSettings):
     openai_api_key: str
     model_name: str = "gpt-3.5-turbo"
     temperature: float = 0.7 # default value, from 0 to 1. Higher value create more creative and randomic answers, lower value create more focused and deterministc answers
+    streaming: bool = False
     _pyclass: Type = ChatOpenAI
 
     model_config = ConfigDict(

core/cat/looking_glass/agent_manager.py

@@ -6,6 +6,7 @@
 from langchain.docstore.document import Document
 from langchain.prompts import PromptTemplate
 from langchain.chains import LLMChain
+from langchain.callbacks.base import BaseCallbackHandler
 from langchain.agents import AgentExecutor, LLMSingleActionAgent
 
 from cat.looking_glass import prompts
@@ -14,6 +15,13 @@
 from cat.log import log
 
 
+class NewTokenHandler(BaseCallbackHandler):
+
+    def __init__(self, cat):
+        self.cat = cat
+    def on_llm_new_token(self, token: str, **kwargs) -> None:
+        self.cat.send_ws_message(token, "chat_token")
+
 
 
 class AgentManager:
@@ -85,7 +93,7 @@ def execute_memory_chain(self, agent_input, prompt_prefix, prompt_suffix):
             verbose=True
         )
 
-        out = memory_chain(agent_input)
+        out = memory_chain(agent_input, callbacks=[NewTokenHandler(self.cat)])
         out["output"] = out["text"]
         del out["text"]
         return out

core/cat/looking_glass/cheshire_cat.py

@@ -23,7 +23,7 @@
 from cat.factory.custom_llm import CustomOpenAI
 
 
-MSG_TYPES = Literal["notification", "chat", "error"]
+MSG_TYPES = Literal["notification", "chat", "error", "chat_token"]
 
 # main class
 class CheshireCat: