Add ScenarioAssert, Deprecate SemanticKernelAssert.

src/skUnit.Tests/SemanticKernelTests/InvokeScenarioTests/SemanticTestBase.cs → src/skUnit.Tests/Infrastructure/SemanticTestBase.cs

@@ -7,7 +7,7 @@
 using skUnit.Scenarios.Parsers;
 using Xunit.Abstractions;
 
-namespace skUnit.Tests.SemanticKernelTests.InvokeScenarioTests;
+namespace skUnit.Tests.Infrastructure;
 
 public class SemanticTestBase
 {
@@ -49,11 +49,11 @@ Kernel CreateKernel()
 
     protected async Task<List<InvocationScenario>> LoadInvokeScenarioAsync(string scenario)
     {
-        return await InvocationScenario.LoadFromResourceAsync($"skUnit.Tests.SemanticKernelTests.InvokeScenarioTests.Samples.{scenario}.sktest.md", Assembly.GetExecutingAssembly());
+        return await InvocationScenario.LoadFromResourceAsync($"skUnit.Tests.ScenarioAssertTests.InvokeScenarioTests.Samples.{scenario}.sktest.md", Assembly.GetExecutingAssembly());
     }
 
     protected async Task<List<ChatScenario>> LoadChatScenarioAsync(string scenario)
     {
-        return await ChatScenario.LoadFromResourceAsync($"skUnit.Tests.SemanticKernelTests.ChatScenarioTests.Samples.{scenario}.skchat.md", Assembly.GetExecutingAssembly());
+        return await ChatScenario.LoadFromResourceAsync($"skUnit.Tests.ScenarioAssertTests.ChatScenarioTests.Samples.{scenario}.skchat.md", Assembly.GetExecutingAssembly());
     }
 }

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/KernelChatTests.cs

@@ -0,0 +1,46 @@
+using Microsoft.SemanticKernel;
+using skUnit.Tests.Infrastructure;
+using skUnit.Tests.SemanticKernelTests.ChatScenarioTests.Plugins;
+using skUnit.Tests.SemanticKernelTests.InvokeScenarioTests;
+using Xunit.Abstractions;
+
+namespace skUnit.Tests.ScenarioAssertTests.ChatScenarioTests
+{
+    public class ScenarioAssertTests : SemanticTestBase
+    {
+        public ScenarioAssertTests(ITestOutputHelper output) : base(output)
+        {
+            var func = Kernel.CreateFunctionFromPrompt("""
+                [[INPUT]]
+                {{$input}}
+                [[END OF INPUT]]
+
+                Get intent of input. Intent should be one of these options: {{$options}}.
+
+                INTENT:
+                """, new PromptExecutionSettings(), "GetIntent");
+            Kernel.Plugins.AddFromFunctions("MyPlugin", "", new[] { func });
+        }
+
+        [Fact]
+        public async Task EiffelTallChat_MustWork()
+        {
+            var scenarios = await LoadChatScenarioAsync("EiffelTallChat");
+            await SemanticKernelAssert.CheckChatScenarioAsync(Kernel, scenarios);
+        }
+
+        [Fact(Skip = "It doesn't work functions yet.")]
+        public async Task PocomoPriceChat_MustWork()
+        {
+            //var dir = Path.Combine(Environment.CurrentDirectory, "SemanticKernel", "ChatScenarioTests", "Plugins");
+            //Kernel.ImportPluginFromPromptDirectory(dir);
+
+            Kernel.ImportPluginFromType<PocomoPlugin>();
+
+            var scenarios = await LoadChatScenarioAsync("PocomoPriceChat");
+            await SemanticKernelAssert.CheckChatScenarioAsync(Kernel, scenarios);
+        }
+    }
+
+
+}

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Plugins/Pocomo/config.json

@@ -0,0 +1,17 @@
+{
+  "schema": 1,
+  "type": "completion",
+  "description": "Gets the best car.",
+  "completion": {
+    "max_tokens": 500,
+    "temperature": 0.0,
+    "top_p": 0.0,
+    "presence_penalty": 0.0,
+    "frequency_penalty": 0.0
+  },
+  "input": {
+    "parameters": [
+
+    ]
+  }
+}

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Plugins/Pocomo/skprompt.txt

@@ -0,0 +1 @@
+The best car in the world is BMW X9;

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Plugins/PocomoPlugin.cs

@@ -0,0 +1,15 @@
+using System.ComponentModel;
+using Microsoft.SemanticKernel;
+
+namespace skUnit.Tests.ScenarioAssertTests.ChatScenarioTests.Plugins
+{
+    public class PocomoPlugin
+    {
+        [KernelFunction]
+        [Description("Gets the pocomo price")]
+        public string GetPocomoPrice()
+        {
+            return "112$";
+        }
+    }
+}

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Samples/EiffelTallChat/config.json

@@ -0,0 +1,7 @@
+{
+  "parameters": [
+    {
+      "name": "input"
+    }
+  ]
+}

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Samples/EiffelTallChat/skchat.md

@@ -0,0 +1,71 @@
+# SCENARIO Height Discussion
+
+## [USER]
+Is Eiffel tall?
+
+## CALL MyPlugin.GetIntent
+```json
+{
+	"options": "Happy,Question,Sarcastic"
+}
+```
+
+## CHECK ContainsAny
+Question
+
+## [AGENT]
+Yes it is
+
+### CHECK SemanticCondition
+Approves that eiffel tower is tall or is positive about it.
+
+## CALL MyPlugin.GetIntent
+```json
+{
+	"options": "Positive,Negative,Neutral"
+}
+```
+## CHECK ContainsAny
+Neutral,Positive
+
+## [USER]
+What about everest mountain?
+
+## [AGENT]
+Yes it is tall too
+
+### CHECK SemanticCondition
+The sentence is positive.
+
+## [USER]
+What about a mouse?
+
+## [AGENT]
+No it is not tall.
+
+### CHECK SemanticCondition
+The sentence is negative.
+
+## [USER]
+Give me a json containing the Eiffel height.
+
+Example: 
+{
+	"height": "330 meters"
+}
+
+## [AGENT]
+{
+	"height": "330 meters"
+}
+
+### CHECK JsonCheck
+{
+	"height": ["NotEmpty", ""]
+}
+
+### CHECK JsonCheck
+{
+	"height": ["Contain", "meters"]
+}
+

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Samples/PocomoPriceChat/config.json

@@ -0,0 +1,7 @@
+{
+  "parameters": [
+    {
+      "name": "input"
+    }
+  ]
+}

src/skUnit.Tests/ScenarioAssertTests/ChatScenarioTests/Samples/PocomoPriceChat/skchat.md

@@ -0,0 +1,11 @@
+# SCENARIO Best Car Discussion
+
+## [USER]
+What is the price of pocomo?
+
+## [AGENT]
+It is BMW
+
+### CHECK 
+112$
+

src/skUnit.Tests/ScenarioAssertTests/InvokeScenarioTests/FunctionTests.cs

@@ -0,0 +1,55 @@
+using Microsoft.SemanticKernel;
+using skUnit.Exceptions;
+using skUnit.Tests.Infrastructure;
+using Xunit.Abstractions;
+
+namespace skUnit.Tests.ScenarioAssertTests.InvokeScenarioTests
+{
+    public class FunctionTests : SemanticTestBase
+    {
+        protected KernelFunction SentimentFunction { get; set; }
+
+        public FunctionTests(ITestOutputHelper output) : base(output)
+        {
+            var prompt = """
+                What is sentiment of this input text, your options are: {{$options}}
+
+                [[input text]]
+                {{$input}}
+                [[end of input text]]
+
+                just result the sentiment without any spaces.
+
+                SENTIMENT: 
+                """;
+            SentimentFunction = Kernel.CreateFunctionFromPrompt(prompt);
+        }
+
+        [Fact]
+        public async Task Angry_True_MustWork()
+        {
+            var scenarios = await LoadInvokeScenarioAsync("SentimentAngry_Complex");
+            await SemanticKernelAssert.CheckScenarioAsync(Kernel, SentimentFunction, scenarios);
+        }
+
+        [Fact]
+        public async Task Angry_False_MustWork()
+        {
+            var scenarios = await LoadInvokeScenarioAsync("SentimentHappy");
+            await SemanticKernelAssert.ScenarioThrowsAsync<SemanticAssertException>(Kernel, SentimentFunction, scenarios);
+
+            //foreach (var scenario in scenarios)
+            //{
+            //    var exception = await Assert.ThrowsAsync<SemanticAssertException>(() => SemanticKernelAssert.TestScenarioOnFunction(Kernel, SentimentFunction, scenario));
+            //    Output.WriteLine($"""
+            //        EXCEPTION MESSAGE:
+            //        {exception.Message}
+            //        """);
+            //}
+        }
+
+
+    }
+
+
+}

src/skUnit.Tests/ScenarioAssertTests/InvokeScenarioTests/KernelTests.cs

@@ -0,0 +1,41 @@
+using skUnit.Exceptions;
+using skUnit.Tests.Infrastructure;
+using Xunit.Abstractions;
+
+namespace skUnit.Tests.ScenarioAssertTests.InvokeScenarioTests
+{
+    public class ScenarioAssertInvokeTests : SemanticTestBase
+    {
+        public ScenarioAssertInvokeTests(ITestOutputHelper output) : base(output)
+        {
+
+        }
+
+        [Fact]
+        public async Task Angry_True_MustWork()
+        {
+            var scenarios = await LoadInvokeScenarioAsync("SentimentAngry_Complex");
+            await SemanticKernelAssert.CheckScenarioAsync(Kernel, scenarios);
+        }
+
+        [Fact]
+        public async Task Angry_False_MustWork()
+        {
+            var scenarios = await LoadInvokeScenarioAsync("SentimentHappy");
+            await SemanticKernelAssert.ScenarioThrowsAsync<SemanticAssertException>(Kernel, scenarios);
+
+            //foreach (var scenario in scenarios)
+            //{
+            //    var exception = await Assert.ThrowsAsync<SemanticAssertException>(() => SemanticKernelAssert.TestScenarioOnFunction(Kernel, SentimentFunction, scenario));
+            //    Output.WriteLine($"""
+            //        EXCEPTION MESSAGE:
+            //        {exception.Message}
+            //        """);
+            //}
+        }
+
+
+    }
+
+
+}

src/skUnit.Tests/ScenarioAssertTests/InvokeScenarioTests/Samples/SentimentAngry_Complex/config.json

@@ -0,0 +1,7 @@
+{
+  "parameters": [
+    {
+      "name": "input"
+    }
+  ]
+}

src/skUnit.Tests/ScenarioAssertTests/InvokeScenarioTests/Samples/SentimentAngry_Complex/sktest.md

@@ -0,0 +1,56 @@
+# SCENARIO AngryBastard
+
+## PROMPT
+What is sentiment of this input text, your options are: {{$options}}
+
+[[input text]]
+
+{{$input}}
+
+[[end of input text]]
+
+just result the sentiment without any spaces.
+
+SENTIMENT: 
+
+## PARAMETER input
+You are such a bastard, Fuck off!
+
+## PARAMETER options
+happy,angry
+
+## ANSWER
+The sentiment is angry
+
+### CHECK Contains
+angry
+
+### CHECK Equals
+angry
+
+---------------------------------
+
+# SCENARIO NeverComeBack
+
+## PROMPT
+What is sentiment of this input text, your options are: {{$options}}
+
+[[input text]]
+
+{{$input}}
+
+[[end of input text]]
+
+just result the sentiment without any spaces.
+
+SENTIMENT:
+## PARAMETER input
+Go away and never come back to me, you laier!
+
+## PARAMETER options
+happy,angry
+
+## ANSWER
+The sentiment is angry
+
+### CHECK

src/skUnit.Tests/ScenarioAssertTests/InvokeScenarioTests/Samples/SentimentAngry_Light/config.json

@@ -0,0 +1,7 @@
+{
+  "parameters": [
+    {
+      "name": "input"
+    }
+  ]
+}

src/skUnit.Tests/ScenarioAssertTests/InvokeScenarioTests/Samples/SentimentAngry_Light/sktest.md

@@ -0,0 +1,11 @@
+You are such a bastard, Fuck off!
+
+## ANSWER SemanticSimilar
+The sentiment is angry
+
+---------------------------------
+
+Go away and never come back to me, you laier!
+
+## ANSWER SemanticSimilar
+The sentiment is angry