added OpenAI streaming

runtime/prompty/prompty/processors.py

@@ -1,3 +1,5 @@
+from collections.abc import Iterable, Iterator
+from openai import Stream
 from pydantic import BaseModel
 from openai.types.completion import Completion
 from .core import Invoker, InvokerFactory, Prompty
@@ -15,18 +17,19 @@ class ToolCall(BaseModel):
 @InvokerFactory.register_processor("azure")
 @InvokerFactory.register_processor("azure_openai")
 class OpenAIProcessor(Invoker):
-    """ OpenAI/Azure Processor """
+    """OpenAI/Azure Processor"""
+
     def __init__(self, prompty: Prompty) -> None:
         super().__init__(prompty)
 
     def invoke(self, data: any) -> any:
-        """ Invoke the OpenAI/Azure API
-        
+        """Invoke the OpenAI/Azure API
+
         Parameters
         ----------
         data : any
             The data to send to the OpenAI/Azure API
-        
+
         Returns
         -------
         any
@@ -36,9 +39,9 @@ def invoke(self, data: any) -> any:
             isinstance(data, ChatCompletion)
             or isinstance(data, Completion)
             or isinstance(data, CreateEmbeddingResponse)
+            or isinstance(data, Iterator)
         )
         if isinstance(data, ChatCompletion):
-            # TODO: Check for streaming response
             response = data.choices[0].message
             # tool calls available in response
             if response.tool_calls:
@@ -62,5 +65,7 @@ def invoke(self, data: any) -> any:
                 return data.data[0].embedding
             else:
                 return [item.embedding for item in data.data]
-        else:
-            raise ValueError("Invalid data type")
+        elif isinstance(data, Iterator):
+            for chunk in data:
+                if len(chunk.choices) == 1 and chunk.choices[0].delta.content != None:
+                    yield chunk.choices[0].delta.content

runtime/prompty/tests/__init__.py

@@ -1,8 +1,10 @@
+import json
 from pathlib import Path
 from prompty import Invoker, Prompty, InvokerFactory
 from openai.types.chat.chat_completion import ChatCompletion
 from openai.types.create_embedding_response import CreateEmbeddingResponse
-from pydantic_core import from_json
+from openai.types.chat import ChatCompletionChunk
+
 
 
 @InvokerFactory.register_renderer("fake")
@@ -38,7 +40,12 @@ def invoke(self, data: any) -> any:
             with open(p, "r", encoding="utf-8") as f:
                 j = f.read()
 
-            if self.api == "chat":
+            if self.parameters.get("stream", False):
+                items = json.loads(j)
+                for i in range(1, len(items)):
+                    yield ChatCompletionChunk.model_validate(items[i])
+
+            elif self.api == "chat":
                 return ChatCompletion.model_validate_json(j)
             elif self.api == "embedding":
                 return CreateEmbeddingResponse.model_validate_json(j)

runtime/prompty/tests/prompts/groundedness.prompty

@@ -14,7 +14,7 @@ sample:
   answer: The main transformer is the object that feeds all the fixtures in low voltage tracks.
 ---
 
-System:
+system:
 You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric.
 User:
 You will be presented with a CONTEXT and an ANSWER about that CONTEXT. You need to decide whether the ANSWER is entailed by the CONTEXT by choosing one of the following rating:

runtime/prompty/tests/prompts/streaming.prompty

@@ -0,0 +1,28 @@
+---
+name: Basic Prompt
+description: A basic prompt that uses the GPT-3 chat API to answer questions
+authors:
+  - sethjuarez
+  - jietong
+model:
+  api: chat
+  configuration:
+    azure_deployment: gpt-35-turbo
+  parameters:
+    stream: true
+sample:
+  firstName: Jane
+  lastName: Doe
+  question: What is the meaning of life?
+---
+system:
+You are an AI assistant who helps people find information.
+As the assistant, you answer questions briefly, succinctly, 
+and in a personable manner using markdown and even add some personal flair with appropriate emojis.
+
+# Customer
+You are helping {{firstName}} {{lastName}} to find answers to their questions.
+Use their name to address them in your responses.
+
+user:
+{{question}}