From 1dfb45f6fef91f08ac32b1950660db637b76d140 Mon Sep 17 00:00:00 2001 From: Teresa Hoang <125500434+teresaqhoang@users.noreply.github.com> Date: Thu, 15 Feb 2024 14:01:15 -0500 Subject: [PATCH] .Net: Removing Insufficient functions error (#4983) ### Motivation and Context This PR removes the InsufficientFunctions error message. If the planner doesn't have enough context to create a plan, it should forego a template altogether and return an error string. This will be captured by the `InvalidTemplate` check. ### Description Addresses #4442, which identified that this error handling was so restrictive for most plan types. ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --- .../Handlebars/CreatePlanPrompt.handlebars | 2 +- .../TipsAndInstructions.handlebars | 2 +- .../Handlebars/HandlebarsPlanner.cs | 15 +-------------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPrompt.handlebars b/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPrompt.handlebars index 87d3a11b35ae..dd820361219e 100644 --- a/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPrompt.handlebars +++ b/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPrompt.handlebars @@ -6,4 +6,4 @@ {{> AdditionalContext }} {{> RetryLogic }} -{{> TipsAndInstructions }} \ No newline at end of file +{{> TipsAndInstructions }} diff --git a/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/TipsAndInstructions.handlebars b/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/TipsAndInstructions.handlebars index 3422893ace63..b287362d2f32 100644 --- a/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/TipsAndInstructions.handlebars +++ b/dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/TipsAndInstructions.handlebars @@ -20,7 +20,7 @@ Follow these steps to create one Handlebars template to achieve the goal: - Only use the helpers provided. Any helper not listed is considered hallucinated and must not be used. - Do not invent or assume the existence of any functions not explicitly defined above. 3. What if I Need More Helpers? - - If the goal cannot be fully achieved with the provided helpers or you need a helper not defined, print the following message: "{{insufficientFunctionsErrorMessage}}". + - Stop here if the goal cannot be fully achieved with the provided helpers or you need a helper not defined, and just return a string with an appropriate error message. 4. Keep It Simple:{{#if allowLoops}} - Avoid using loops or block expressions. They are allowed but not always necessary, so try to find a solution that does not use them.{{/if}} - Your template should be intelligent and efficient, avoiding unnecessary complexity or redundant steps. diff --git a/dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlanner.cs b/dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlanner.cs index 602870e5cc82..82dad6153a29 100644 --- a/dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlanner.cs +++ b/dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlanner.cs @@ -72,11 +72,6 @@ public Task CreatePlanAsync(Kernel kernel, string goal, KernelAr private readonly HandlebarsPromptTemplateFactory _templateFactory; - /// - /// Error message if kernel does not contain sufficient functions to create a plan. - /// - private const string InsufficientFunctionsError = "Additional helpers or information may be required"; - private async Task CreatePlanCoreAsync(Kernel kernel, string goal, KernelArguments? arguments, CancellationToken cancellationToken = default) { // Get CreatePlan prompt template @@ -89,17 +84,10 @@ private async Task CreatePlanCoreAsync(Kernel kernel, string goa var chatCompletionService = kernel.GetRequiredService(); var completionResults = await chatCompletionService.GetChatMessageContentAsync(chatMessages, executionSettings: this._options.ExecutionSettings, cancellationToken: cancellationToken).ConfigureAwait(false); - // Check if plan could not be created due to insufficient functions - if (completionResults.Content is not null && completionResults.Content.IndexOf(InsufficientFunctionsError, StringComparison.OrdinalIgnoreCase) >= 0) - { - var functionNames = availableFunctions.ToList().Select(func => $"{func.PluginName}{this._templateFactory.NameDelimiter}{func.Name}"); - throw new KernelException($"[{HandlebarsPlannerErrorCodes.InsufficientFunctionsForGoal}] Unable to create plan for goal with available functions.\nGoal: {goal}\nAvailable Functions: {string.Join(", ", functionNames)}\nPlanner output:\n{completionResults}"); - } - Match match = Regex.Match(completionResults.Content, @"```\s*(handlebars)?\s*(.*)\s*```", RegexOptions.Singleline); if (!match.Success) { - throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Could not find the plan in the results\nPlanner output:\n{completionResults}"); + throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Could not find the plan in the results. Additional helpers or input may be required.\n\nPlanner output:\n{completionResults}"); } var planTemplate = match.Groups[2].Value.Trim(); @@ -236,7 +224,6 @@ private async Task GetHandlebarsTemplateAsync( { "goal", goal }, { "predefinedArguments", predefinedArgumentsWithTypes}, { "nameDelimiter", this._templateFactory.NameDelimiter}, - { "insufficientFunctionsErrorMessage", InsufficientFunctionsError}, { "allowLoops", this._options.AllowLoops }, { "complexTypeDefinitions", complexParameterTypes.Count > 0 && complexParameterTypes.Any(p => p.IsComplex) ? complexParameterTypes.Where(p => p.IsComplex) : null}, { "complexSchemaDefinitions", complexParameterSchemas.Count > 0 ? complexParameterSchemas : null},