feat: parse response_format from span into playground store

app/src/pages/playground/constants.tsx

@@ -26,6 +26,8 @@ export const MODEL_CONFIG_PARSING_ERROR =
   "Unable to parse model config, expected llm.model_name to be present.";
 export const MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR =
   "Unable to parse model config, expected llm.invocation_parameters json string to be present.";
+export const MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR =
+  "Unable to parse invocation parameters response_format, expected llm.invocation_parameters.response_format to be a well formed json object or undefined.";
 // TODO(parker / apowell) - adjust this error message with anthropic support https://github.com/Arize-ai/phoenix/issues/5100
 export const TOOLS_PARSING_ERROR =
   "Unable to parse tools, expected tools to be an array of valid OpenAI tools.";

app/src/pages/playground/playgroundUtils.ts

@@ -22,6 +22,7 @@ import {
   INPUT_MESSAGES_PARSING_ERROR,
   MODEL_CONFIG_PARSING_ERROR,
   MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR,
+  MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR,
   modelProviderToModelPrefixMap,
   OUTPUT_MESSAGES_PARSING_ERROR,
   OUTPUT_VALUE_PARSING_ERROR,
@@ -32,13 +33,15 @@ import { InvocationParameter } from "./InvocationParametersForm";
 import {
   chatMessageRolesSchema,
   chatMessagesSchema,
+  JsonObjectSchema,
   llmInputMessageSchema,
   llmOutputMessageSchema,
   LlmToolSchema,
   llmToolSchema,
   MessageSchema,
   modelConfigSchema,
   modelConfigWithInvocationParametersSchema,
+  modelConfigWithResponseFormatSchema,
   outputSchema,
 } from "./schemas";
 import { PlaygroundSpan } from "./spanPlaygroundPageLoader";
@@ -261,6 +264,21 @@ export function getModelInvocationParametersFromAttributes(
   };
 }
 
+export function getResponseFormatFromAttributes(parsedAttributes: unknown) {
+  const { success, data } =
+    modelConfigWithResponseFormatSchema.safeParse(parsedAttributes);
+  if (!success) {
+    return {
+      responseFormat: undefined,
+      parsingErrors: [MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR],
+    };
+  }
+  return {
+    responseFormat: data.llm.invocation_parameters.response_format,
+    parsingErrors: [],
+  };
+}
+
 /**
  * Processes the tools from the span attributes into OpenAI tools to be used in the playground
  * @param tools tools from the span attributes
@@ -353,6 +371,8 @@ export function transformSpanAttributesToPlaygroundInstance(
     // See https://github.com/Arize-ai/phoenix/issues/5235
     []
   );
+  const { responseFormat, parsingErrors: responseFormatParsingErrors } =
+    getResponseFormatFromAttributes(parsedAttributes);
 
   // Merge invocation parameters into model config, if model config is present
   modelConfig =
@@ -382,13 +402,15 @@ export function transformSpanAttributesToPlaygroundInstance(
       output,
       spanId: span.id,
       tools: tools ?? basePlaygroundInstance.tools,
+      responseFormat,
     },
     parsingErrors: [
       ...messageParsingErrors,
       ...outputParsingErrors,
       ...modelConfigParsingErrors,
       ...toolsParsingErrors,
       ...invocationParametersParsingErrors,
+      ...responseFormatParsingErrors,
     ],
   };
 }
@@ -483,7 +505,7 @@ export const constrainInvocationParameterInputsToDefinition = (
  */
 export const transformInvocationParametersFromAttributesToInvocationParameterInputs =
   (
-    invocationParameters: Record<string, string | number | boolean | string[]>,
+    invocationParameters: JsonObjectSchema,
     modelSupportedInvocationParameters: InvocationParameter[]
   ): InvocationParameterInput[] => {
     return Object.entries(invocationParameters)

app/src/pages/playground/schemas.ts

@@ -115,10 +115,18 @@ const jsonLiteralSchema: z.ZodType<Json> = z.lazy(() =>
   ])
 );
 
+export type JsonLiteralSchema = z.infer<typeof jsonLiteralSchema>;
+
+const jsonObjectSchema: z.ZodType<{ [key: string]: Json }> = z.lazy(() =>
+  z.record(jsonLiteralSchema)
+);
+
+export type JsonObjectSchema = z.infer<typeof jsonObjectSchema>;
+
 /**
  * Model generic invocation parameters schema in zod.
  */
-const invocationParameterSchema = jsonLiteralSchema;
+const invocationParameterSchema = jsonObjectSchema;
 
 /**
  * The type of the invocation parameters schema
@@ -178,6 +186,17 @@ export const modelConfigWithInvocationParametersSchema = z.object({
   }),
 });
 
+export const modelConfigWithResponseFormatSchema = z.object({
+  [SemanticAttributePrefixes.llm]: z.object({
+    [LLMAttributePostfixes.invocation_parameters]:
+      stringToInvocationParametersSchema.pipe(
+        z.object({
+          response_format: jsonObjectSchema.optional(),
+        })
+      ),
+  }),
+});
+
 /**
  *  The zod schema for llm.tools.{i}.tool.json_schema attribute
  *  This will be a json string parsed into an object

app/src/store/playground/types.ts

@@ -1,6 +1,7 @@
 import { TemplateLanguage } from "@phoenix/components/templateEditor/types";
 import { InvocationParameterInput } from "@phoenix/pages/playground/__generated__/PlaygroundOutputSubscription.graphql";
 import { InvocationParameter } from "@phoenix/pages/playground/InvocationParametersForm";
+import { JsonObjectSchema } from "@phoenix/pages/playground/schemas";
 import { OpenAIToolCall, OpenAIToolDefinition } from "@phoenix/schemas";
 
 import { ModelConfigByProvider } from "../preferencesStore";
@@ -111,6 +112,7 @@ export interface PlaygroundInstance {
    * @default "auto"
    */
   toolChoice: ToolChoice;
+  responseFormat?: JsonObjectSchema;
   input: PlaygroundInput;
   model: ModelConfig;
   output?: ChatMessage[] | string;