Merge pull request #492 from cheshire-cat-ai/develop

1.2.0 - Streaming & update dependencies

core/Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get -y update && apt-get install -y curl build-essential fastjar libmagi
 
 ### ADMIN (static build) ###
 WORKDIR /admin
-RUN curl -sL https://github.com/cheshire-cat-ai/admin-vue/releases/download/Admin/release.zip | jar -xv
+RUN curl -sL https://github.com/cheshire-cat-ai/admin-vue/releases/download/Admin/develop.zip | jar -xv
 
 ### INSTALL PYTHON DEPENDENCIES (Core and Plugins) ###
 WORKDIR /app

core/cat/factory/llm.py

@@ -97,6 +97,7 @@ class LLMOpenAIChatConfig(LLMSettings):
     openai_api_key: str
     model_name: str = "gpt-3.5-turbo"
     temperature: float = 0.7 # default value, from 0 to 1. Higher value create more creative and randomic answers, lower value create more focused and deterministc answers
+    streaming: bool = True
     _pyclass: Type = ChatOpenAI
 
     model_config = ConfigDict(
@@ -112,6 +113,7 @@ class LLMOpenAIConfig(LLMSettings):
     openai_api_key: str
     model_name: str = "gpt-3.5-turbo-instruct" # used instead of text-davinci-003 since it deprecated
     temperature: float = 0.7 # default value, from 0 to 1. Higher value create more creative and randomic answers, lower value create more focused and deterministc answers
+    streaming: bool = True
     _pyclass: Type = OpenAI
 
     model_config = ConfigDict(

core/cat/looking_glass/agent_manager.py

@@ -9,13 +9,12 @@
 from langchain.agents import AgentExecutor, LLMSingleActionAgent
 
 from cat.looking_glass import prompts
+from cat.looking_glass.callbacks import NewTokenHandler
 from cat.looking_glass.output_parser import ToolOutputParser
 from cat.utils import verbal_timedelta
 from cat.log import log
 
 
-
-
 class AgentManager:
     """Manager of Langchain Agent.
 
@@ -46,7 +45,11 @@ def execute_tool_agent(self, agent_input, allowed_tools):
         )
 
         # main chain
-        agent_chain = LLMChain(prompt=prompt, llm=self.cat._llm, verbose=True)
+        agent_chain = LLMChain(
+            prompt=prompt,
+            llm=self.cat._llm,
+            verbose=True
+        )
 
         # init agent
         agent = LLMSingleActionAgent(
@@ -85,7 +88,7 @@ def execute_memory_chain(self, agent_input, prompt_prefix, prompt_suffix):
             verbose=True
         )
 
-        out = memory_chain(agent_input)
+        out = memory_chain(agent_input, callbacks=[NewTokenHandler(self.cat)])
         out["output"] = out["text"]
         del out["text"]
         return out

core/cat/looking_glass/callbacks.py

@@ -0,0 +1,11 @@
+
+from langchain.callbacks.base import BaseCallbackHandler
+
+
+class NewTokenHandler(BaseCallbackHandler):
+
+    def __init__(self, cat):
+        self.cat = cat
+
+    def on_llm_new_token(self, token: str, **kwargs) -> None:
+        self.cat.send_ws_message(token, "chat_token")

core/cat/looking_glass/cheshire_cat.py

@@ -2,28 +2,28 @@
 from copy import deepcopy
 import traceback
 from typing import Literal, get_args
-import langchain
 import os
 import asyncio
+import langchain
+from langchain.llms import Cohere, OpenAI, AzureOpenAI, HuggingFaceTextGenInference, HuggingFaceHub
+from langchain.chat_models import ChatOpenAI, AzureChatOpenAI
+from langchain.base_language import BaseLanguageModel
+
 from cat.log import log
+from cat.db import crud
 from cat.db.database import Database
 from cat.rabbit_hole import RabbitHole
 from cat.mad_hatter.mad_hatter import MadHatter
 from cat.memory.working_memory import WorkingMemoryList
 from cat.memory.long_term_memory import LongTermMemory
 from cat.looking_glass.agent_manager import AgentManager
-
-# TODO: natural language dependencies; move to another file
+from cat.looking_glass.callbacks import NewTokenHandler
 import cat.factory.llm as llms
 import cat.factory.embedder as embedders
-from cat.db import crud
-from langchain.llms import Cohere, OpenAI, OpenAIChat, AzureOpenAI, HuggingFaceTextGenInference, HuggingFaceHub
-from langchain.chat_models import ChatOpenAI, AzureChatOpenAI
-from langchain.base_language import BaseLanguageModel
 from cat.factory.custom_llm import CustomOpenAI
 
 
-MSG_TYPES = Literal["notification", "chat", "error"]
+MSG_TYPES = Literal["notification", "chat", "error", "chat_token"]
 
 # main class
 class CheshireCat:
@@ -163,7 +163,7 @@ def get_language_embedder(self) -> embedders.EmbedderSettings:
             return embedder
 
         # OpenAI embedder
-        if type(self._llm) in [OpenAI, OpenAIChat]:
+        if type(self._llm) in [OpenAI, ChatOpenAI]:
             embedder = embedders.EmbedderOpenAIConfig.get_embedder_from_config(
                 {
                     "openai_api_key": self._llm.openai_api_key,
@@ -315,7 +315,7 @@ def recall_relevant_memories_to_working_memory(self):
         # hook to modify/enrich retrieved memories
         self.mad_hatter.execute_hook("after_cat_recalls_memories")
 
-    def llm(self, prompt: str) -> str:
+    def llm(self, prompt: str, chat: bool = False, stream: bool = False) -> str:
         """Generate a response using the LLM model.
 
         This method is useful for generating a response with both a chat and a completion model using the same syntax
@@ -331,13 +331,19 @@ def llm(self, prompt: str) -> str:
             The generated response.
 
         """
+
+        # should we stream the tokens?
+        callbacks = []
+        if stream:
+            callbacks.append( NewTokenHandler(self) )
+
         # Check if self._llm is a completion model and generate a response
         if isinstance(self._llm, langchain.llms.base.BaseLLM):
-            return self._llm(prompt)
+            return self._llm(prompt, callbacks=callbacks)
 
         # Check if self._llm is a chat model and call it as a completion model
         if isinstance(self._llm, langchain.chat_models.base.BaseChatModel):
-            return self._llm.call_as_llm(prompt)
+            return self._llm.call_as_llm(prompt, callbacks=callbacks)
 
     def send_ws_message(self, content: str, msg_type: MSG_TYPES = "notification"):
         """Send a message via websocket.

core/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "Cheshire-Cat"
 description = "Production ready AI assistant framework"
-version = "1.1.2"
+version = "1.2.0"
 requires-python = ">=3.10"
 license = { file="LICENSE" }
 authors = [