plumb through span tools to playground

app/src/pages/playground/PlaygroundOutput.tsx

@@ -34,6 +34,7 @@ interface PlaygroundOutputProps extends PlaygroundInstanceProps {}
 
 function PlaygroundOutputMessage({ message }: { message: ChatMessage }) {
   const { role, content, toolCalls } = message;
+  console.log("test--", message);
   const styles = useChatMessageStyles(role);
 
   return (

app/src/pages/playground/constants.tsx

@@ -26,6 +26,9 @@ export const MODEL_CONFIG_PARSING_ERROR =
   "Unable to parse model config, expected llm.model_name to be present.";
 export const MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR =
   "Unable to parse model config, expected llm.invocation_parameters json string to be present.";
+// TODO(parker / apowell) - adjust this error message with anthropic support https://github.com/Arize-ai/phoenix/issues/5100
+export const TOOLS_PARSING_ERROR =
+  "Unable to parse tools, expected tools to be an array of valid OpenAI tools.";
 
 export const modelProviderToModelPrefixMap: Record<ModelProvider, string[]> = {
   AZURE_OPENAI: [],

app/src/pages/playground/playgroundUtils.ts

@@ -4,11 +4,14 @@ import {
   DEFAULT_CHAT_ROLE,
   DEFAULT_MODEL_PROVIDER,
 } from "@phoenix/constants/generativeConstants";
-import { ModelConfig, PlaygroundInstance } from "@phoenix/store";
 import {
   ChatMessage,
   createPlaygroundInstance,
   generateMessageId,
+  generateToolId,
+  ModelConfig,
+  OpenAITool,
+  PlaygroundInstance,
 } from "@phoenix/store";
 import { assertUnreachable } from "@phoenix/typeUtils";
 import { safelyParseJSON } from "@phoenix/utils/jsonUtils";
@@ -22,12 +25,15 @@ import {
   OUTPUT_MESSAGES_PARSING_ERROR,
   OUTPUT_VALUE_PARSING_ERROR,
   SPAN_ATTRIBUTES_PARSING_ERROR,
+  TOOLS_PARSING_ERROR,
 } from "./constants";
 import {
   chatMessageRolesSchema,
   chatMessagesSchema,
   llmInputMessageSchema,
   llmOutputMessageSchema,
+  LlmToolSchema,
+  llmToolSchema,
   MessageSchema,
   modelConfigSchema,
   modelConfigWithInvocationParametersSchema,
@@ -229,6 +235,47 @@ export function getModelConfigFromAttributes(parsedAttributes: unknown): {
   return { modelConfig: null, parsingErrors: [MODEL_CONFIG_PARSING_ERROR] };
 }
 
+/**
+ * Processes the tools from the span attributes into OpenAI tools to be used in the playground
+ * @param tools tools from the span attributes
+ * @returns playground OpenAI tools
+ */
+function processAttributeTools(tools: LlmToolSchema): OpenAITool[] {
+  return (tools?.llm?.tools ?? [])
+    .map((tool) => {
+      if (tool?.tool == null) {
+        return null;
+      }
+      return {
+        id: generateToolId(),
+        definition: tool.tool.json_schema,
+      };
+    })
+    .filter((tool): tool is NonNullable<typeof tool> => tool != null);
+}
+
+/**
+ * Attempts to get llm.tools from the span attributes.
+ * @param parsedAttributes the JSON parsed span attributes
+ * @returns the tools from the span attributes
+ */
+export function getToolsFromAttributes(
+  parsedAttributes: unknown
+):
+  | { tools: OpenAITool[]; parsingErrors: never[] }
+  | { tools: null; parsingErrors: string[] } {
+  const { data, success } = llmToolSchema.safeParse(parsedAttributes);
+
+  if (!success) {
+    return { tools: null, parsingErrors: [TOOLS_PARSING_ERROR] };
+  }
+  // If there are no tools or llm attributes, we don't want to return parsing errors, it just means the span didn't have tools
+  if (data?.llm?.tools == null) {
+    return { tools: null, parsingErrors: [] };
+  }
+  return { tools: processAttributeTools(data), parsingErrors: [] };
+}
+
 /**
  * Takes a  {@link PlaygroundSpan|Span} and attempts to transform it's attributes into various fields on a {@link PlaygroundInstance}.
  * @param span the {@link PlaygroundSpan|Span} to transform into a playground instance
@@ -264,7 +311,10 @@ export function transformSpanAttributesToPlaygroundInstance(
   const { modelConfig, parsingErrors: modelConfigParsingErrors } =
     getModelConfigFromAttributes(parsedAttributes);
 
-  // TODO(parker): add support for tools, variables, and input / output variants
+  const { tools, parsingErrors: toolsParsingErrors } =
+    getToolsFromAttributes(parsedAttributes);
+
+  // TODO(parker): add support for prompt template variables
   // https://github.com/Arize-ai/phoenix/issues/4886
   return {
     playgroundInstance: {
@@ -278,11 +328,13 @@ export function transformSpanAttributesToPlaygroundInstance(
             }
           : basePlaygroundInstance.template,
       output,
+      tools: tools ?? basePlaygroundInstance.tools,
     },
     parsingErrors: [
       ...messageParsingErrors,
       ...outputParsingErrors,
       ...modelConfigParsingErrors,
+      ...toolsParsingErrors,
     ],
   };
 }

app/src/pages/playground/schemas.ts

@@ -4,10 +4,13 @@ import {
   LLMAttributePostfixes,
   MessageAttributePostfixes,
   SemanticAttributePrefixes,
+  SemanticConventions,
+  ToolAttributePostfixes,
 } from "@arizeai/openinference-semantic-conventions";
 
+import { openAIToolCallSchema, openAIToolSchema } from "@phoenix/schemas";
 import { ChatMessage } from "@phoenix/store";
-import { Mutable, schemaForType } from "@phoenix/typeUtils";
+import { isObject, Mutable, schemaForType } from "@phoenix/typeUtils";
 import { safelyParseJSON } from "@phoenix/utils/jsonUtils";
 
 import { InvocationParameters } from "./__generated__/PlaygroundOutputSubscription.graphql";
@@ -39,7 +42,7 @@ const toolCallSchema = z
 const messageSchema = z.object({
   [SemanticAttributePrefixes.message]: z.object({
     [MessageAttributePostfixes.role]: z.string(),
-    [MessageAttributePostfixes.content]: z.string(),
+    [MessageAttributePostfixes.content]: z.string().optional(),
     [MessageAttributePostfixes.tool_calls]: z.array(toolCallSchema).optional(),
   }),
 });
@@ -91,7 +94,10 @@ const chatMessageSchema = schemaForType<ChatMessage>()(
   z.object({
     id: z.number(),
     role: chatMessageRolesSchema,
-    content: z.string(),
+    // Tool call messages may not have content
+    content: z.string().optional(),
+    toolCallId: z.string().optional(),
+    toolCalls: z.array(openAIToolCallSchema).optional(),
   })
 );
 
@@ -182,6 +188,67 @@ export const modelConfigWithInvocationParametersSchema = z.object({
   }),
 });
 
+/**
+ *  The zod schema for llm.tools.{i}.tool.json_schema attribute
+ *  This will be a json string but should be parsed as an object
+ */
+export const toolJSONSchemaSchema = z
+  .string()
+  .transform((s, ctx) => {
+    const { json } = safelyParseJSON(s);
+
+    if (json == null || !isObject(json)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "The tool JSON schema must be a valid JSON object",
+      });
+      return z.NEVER;
+    }
+    return json;
+  })
+  // TODO(parker / apowell) - adjust this transformation with anthropic tool support https://github.com/Arize-ai/phoenix/issues/5100
+  .transform((o, ctx) => {
+    const { data, success } = openAIToolSchema.safeParse(o);
+
+    if (!success) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "The tool JSON schema must be a valid OpenAI tool schema",
+      });
+      return z.NEVER;
+    }
+    return data;
+  });
+
+/**
+ * The zod schema for llm.tools
+ * @see {@link  https://github.com/Arize-ai/openinference/blob/main/spec/semantic_conventions.md|Semantic Conventions}
+ * Note there are other attributes that can be on llm.tools.{i}.tool, namely description, name, and parameters
+ * however, these are encompassed by the json schema in some cases and calls to api's using tools in a non json format
+ * is not supported in the playground yet
+ */
+export const llmToolSchema = z
+  .object({
+    [SemanticAttributePrefixes.llm]: z
+      .object({
+        [LLMAttributePostfixes.tools]: z
+          .array(
+            z
+              .object({
+                [SemanticAttributePrefixes.tool]: z.object({
+                  [ToolAttributePostfixes.json_schema]: toolJSONSchemaSchema,
+                }),
+              })
+              .optional()
+          )
+          .optional(),
+      })
+      .optional(),
+  })
+  .optional();
+
+export type LlmToolSchema = z.infer<typeof llmToolSchema>;
+
 /**
  * Default set of invocation parameters for all providers and models.
  */

app/src/store/playground/types.ts

@@ -31,6 +31,7 @@ export type PlaygroundInputMode = "manual" | "dataset";
 export type ChatMessage = {
   id: number;
   role: ChatMessageRole;
+  // Tool call messages may not have content
   content?: string;
   toolCalls?: OpenAIToolCall[];
   toolCallId?: string;

src/phoenix/server/api/subscriptions.py

@@ -624,7 +624,7 @@ def _llm_output_messages(
     yield f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_ROLE}", "assistant"
     if content := "".join(chunk.content for chunk in text_chunks):
         yield f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_CONTENT}", content
-    for tool_call_index, tool_call_chunks_ in tool_call_chunks.items():
+    for tool_call_index, (_tool_call_id, tool_call_chunks_) in enumerate(tool_call_chunks.items()):
         if tool_call_chunks_ and (name := tool_call_chunks_[0].function.name):
             yield (
                 f"{LLM_OUTPUT_MESSAGES}.0.{MESSAGE_TOOL_CALLS}.{tool_call_index}.{TOOL_CALL_FUNCTION_NAME}",