Merge pull request #396 from Lombiq/issue/OSOE-365

OSOE-365: Support for running tests without a browser

Lombiq.Tests.UI.Samples/Tests/SecurityScanningTests.cs

@@ -48,7 +48,10 @@ public SecurityScanningTests(ITestOutputHelper testOutputHelper)
     // will fail the scan, but don't worry! You'll get a nice report about the findings in the failure dump.
     [Fact]
     public Task BasicSecurityScanShouldPass() =>
-        ExecuteTestAfterSetupAsync(
+        // Note how we use a method that doesn't launch a browser. Security scanning happens fully in ZAP, and doesn't
+        // use the browser launched by the UI Testing Toolbox. Not starting a browser for the test makes it a bit
+        // faster. However, you can opt to launch a browser to prepare the app for security scanning if necessary.
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
             context => context.RunAndAssertBaselineSecurityScanAsync(),
             // You should configure the assertion that checks the app logs to accept some common cases that only should
             // appear during security scanning. If you launch a full scan, this is automatically configured by the
@@ -74,15 +77,15 @@ public Task BasicSecurityScanShouldPass() =>
     //   are only present to illustrate the type of adjustments you may want for your own site.
     [Fact]
     public Task SecurityScanWithCustomConfigurationShouldPass() =>
-        ExecuteTestAfterSetupAsync(
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
             context => context.RunAndAssertBaselineSecurityScanAsync(
                 configuration => configuration
                     ////.UseAjaxSpider() // This is quite slow so just showing you here but not running it.
                     .ExcludeUrlWithRegex(".*blog.*")
                     .DisablePassiveScanRule(10020, "The response does not include either Content-Security-Policy with 'frame-ancestors' directive.")
                     .DisableScanRuleForUrlWithRegex(".*/about", 10038, "Content Security Policy (CSP) Header Not Set")
                     .SignIn(),
-                sarifLog => sarifLog.Runs[0].Results.Count.ShouldBe(1)),
+                sarifLog => sarifLog.Runs[0].Results.Count.ShouldBe(0)),
             changeConfiguration: configuration => configuration.UseAssertAppLogsForSecurityScan());
 
     // Let's get low-level into ZAP's configuration now. While the .NET configuration API of the Lombiq UI Testing
@@ -105,7 +108,7 @@ public Task SecurityScanWithCustomConfigurationShouldPass() =>
     // customize them if something you need is not surfaced as configuration.
     [Fact]
     public Task SecurityScanWithCustomAutomationFrameworkPlanShouldPass() =>
-        ExecuteTestAfterSetupAsync(
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
             context => context.RunAndAssertSecurityScanAsync(
                 "Tests/CustomZapAutomationFrameworkPlan.yml",
                 configuration => configuration

Lombiq.Tests.UI.Tests.UI/TestCases/TimeoutTestCases.cs

@@ -7,7 +7,7 @@ public static class TimeoutTestCases
 {
     public static Task TestRunTimeoutShouldThrowAsync(
         ExecuteTestAfterSetupAsync executeTestAfterSetupAsync,
-        Browser browser = default) =>
+        Browser browser = Browser.None) =>
         Should.ThrowAsync(
             async () => await executeTestAfterSetupAsync(
                 context => Task.Delay(TimeSpan.FromSeconds(1)),

Lombiq.Tests.UI/Extensions/IWebApplicationInstanceExtensions.cs

@@ -39,6 +39,15 @@ public static async Task UsingScopeAsync(
 
         try
         {
+            // If there's no Default shell settings then the shell host hasn't been initialized yet. This can happen if
+            // no request hit the app yet.
+            var defaultShellSettingExist = shellHost.TryGetSettings("Default", out var _);
+
+            if (!defaultShellSettingExist)
+            {
+                await shellHost.InitializeAsync();
+            }
+
             // Injecting a fake HttpContext is required for many things, but it needs to happen before UsingAsync()
             // below to avoid NullReferenceExceptions in
             // OrchardCore.Recipes.Services.RecipeEnvironmentFeatureProvider.PopulateEnvironmentAsync. Migrations

Lombiq.Tests.UI/Extensions/VerificationUITestContextExtensions.cs

@@ -20,7 +20,7 @@ public static async Task AssertLogsAsync(this UITestContext context)
         var configuration = context.Configuration;
         var testOutputHelper = configuration.TestOutputHelper;
 
-        await context.UpdateHistoricBrowserLogAsync();
+        if (context.IsBrowserRunning) await context.UpdateHistoricBrowserLogAsync();
 
         try
         {
@@ -34,16 +34,19 @@ public static async Task AssertLogsAsync(this UITestContext context)
             throw;
         }
 
-        try
-        {
-            configuration.AssertBrowserLog?.Invoke(context.HistoricBrowserLog);
-        }
-        catch (Exception)
+        if (context.IsBrowserRunning)
         {
-            testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine);
-            testOutputHelper.WriteLine(context.HistoricBrowserLog.ToFormattedString());
-
-            throw;
+            try
+            {
+                configuration.AssertBrowserLog?.Invoke(context.HistoricBrowserLog);
+            }
+            catch (Exception)
+            {
+                testOutputHelper.WriteLine("Browser logs: " + Environment.NewLine);
+                testOutputHelper.WriteLine(context.HistoricBrowserLog.ToFormattedString());
+
+                throw;
+            }
         }
     }
 }

Lombiq.Tests.UI/OrchardCoreUITestBase.cs

@@ -148,6 +148,53 @@ protected virtual Task ExecuteMultiSizeTestAfterSetupAsync(
             browser,
             changeConfigurationAsync);
 
+    protected virtual Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(testAsync, default, changeConfiguration);
+
+    protected Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(testAsync, default, changeConfigurationAsync);
+
+    protected virtual Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAfterBrowserSetupWithoutBrowserAsync(testAsync, changeConfiguration.AsCompletedTask());
+
+    protected Task ExecuteTestAfterBrowserSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Browser browser,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAfterSetupWithoutBrowserAsync(testAsync, async configuration =>
+        {
+            configuration.SetupConfiguration.BeforeSetup = configuration =>
+            {
+                configuration.BrowserConfiguration.Browser = browser;
+                return Task.CompletedTask;
+            };
+
+            configuration.SetupConfiguration.AfterSetup = configuration =>
+            {
+                configuration.BrowserConfiguration.Browser = Browser.None;
+                return Task.CompletedTask;
+            };
+
+            if (changeConfigurationAsync != null) await changeConfigurationAsync(configuration);
+        });
+
+    protected virtual Task ExecuteTestAfterSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>
+        ExecuteTestAfterSetupWithoutBrowserAsync(testAsync, changeConfiguration.AsCompletedTask());
+
+    protected Task ExecuteTestAfterSetupWithoutBrowserAsync(
+        Func<UITestContext, Task> testAsync,
+        Func<OrchardCoreUITestExecutorConfiguration, Task> changeConfigurationAsync) =>
+        ExecuteTestAfterSetupAsync(testAsync, Browser.None, changeConfigurationAsync);
+
     protected virtual Task ExecuteTestAfterSetupAsync(
         Action<UITestContext> test,
         Action<OrchardCoreUITestExecutorConfiguration> changeConfiguration = null) =>

Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/Baseline.yml

@@ -65,6 +65,25 @@ jobs:
         threshold: high
     name: passiveScan-config
     type: passiveScan-config
+  - alertFilters:
+      # The Error shortcut shows an error page by design.
+      - ruleId: 90022
+        ruleName: Application Error Disclosure (90022)
+        context: ''
+        newRisk: False Positive
+        parameter: ''
+        parameterRegex: false
+        url: .*/Lombiq.Tests.UI.Shortcuts/Error/Index
+        urlRegex: true
+        attack: ''
+        attackRegex: false
+        evidence: ''
+        evidenceRegex: false
+        methods: []
+    parameters:
+      deleteGlobalAlerts: false
+    name: alertFilter
+    type: alertFilter
   - parameters: {}
     name: spider
     type: spider

Lombiq.Tests.UI/SecurityScanning/AutomationFrameworkPlans/FullScan.yml

@@ -81,6 +81,20 @@ jobs:
         evidence: ''
         evidenceRegex: false
         methods: []
+      # The Error shortcut shows an error page by design.
+      - ruleId: 90022
+        ruleName: Application Error Disclosure (90022)
+        context: ''
+        newRisk: False Positive
+        parameter: ''
+        parameterRegex: false
+        url: .*/Lombiq.Tests.UI.Shortcuts/Error/Index
+        urlRegex: true
+        attack: ''
+        attackRegex: false
+        evidence: ''
+        evidenceRegex: false
+        methods: []
     parameters:
       deleteGlobalAlerts: false
     name: alertFilter

Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs

@@ -236,7 +236,7 @@ public static Task<SecurityScanResult> RunSecurityScanAsync(
         Action<SecurityScanConfiguration> configure = null)
     {
         var configuration = new SecurityScanConfiguration()
-            .StartAtUri(context.GetCurrentUri());
+            .StartAtUri(context.IsBrowserRunning ? context.GetCurrentUri() : context.TestStartUri);
 
         // By default ignore /vendor/ or /vendors/ URLs. This is case-insensitive. We have no control over them, and
         // they may contain several false positives (e.g. in font-awesome).

Lombiq.Tests.UI/Services/AtataFactory.cs

@@ -33,7 +33,6 @@ public static async Task<AtataScope> StartAtataScopeAsync(
         var browserConfiguration = configuration.BrowserConfiguration;
 
         var builder = AtataContext.Configure()
-            .UseDriver(await CreateDriverAsync(browserConfiguration, timeoutConfiguration, testOutputHelper))
             .UseBaseUrl(baseUri.ToString())
             .UseCulture(browserConfiguration.AcceptLanguage.ToString())
             .UseTestName(configuration.AtataConfiguration.TestName)
@@ -42,6 +41,17 @@ public static async Task<AtataScope> StartAtataScopeAsync(
             .PageSnapshots.UseCdpOrPageSourceStrategy() // #spell-check-ignore-line
             .UseArtifactsPathTemplate(contextId); // Necessary to prevent long paths, an issue under Windows.
 
+        if (configuration.BrowserConfiguration.Browser != Browser.None)
+        {
+            builder
+                .UseDriverInitializationStage(AtataContextDriverInitializationStage.OnDemand)
+                .UseDriver(await CreateDriverFactoryAsync(browserConfiguration, timeoutConfiguration, testOutputHelper));
+        }
+        else
+        {
+            builder.UseDriverInitializationStage(AtataContextDriverInitializationStage.None);
+        }
+
         builder.LogConsumers.AddDebug();
         builder.LogConsumers.Add(new TestOutputLogConsumer(testOutputHelper));
 
@@ -55,15 +65,11 @@ public static void SetupShellCliCommandFactory() =>
             .UseCmdForWindows()
             .UseForOtherOS(new BashShellCliCommandFactory("-login"));
 
-    private static async Task<IWebDriver> CreateDriverAsync(
+    private static async Task<Func<IWebDriver>> CreateDriverFactoryAsync(
         BrowserConfiguration browserConfiguration,
         TimeoutConfiguration timeoutConfiguration,
         ITestOutputHelper testOutputHelper)
     {
-        Task<T> FromAsync<T>(Func<BrowserConfiguration, TimeSpan, Task<T>> factory)
-            where T : IWebDriver =>
-            factory(browserConfiguration, timeoutConfiguration.PageLoadTimeout);
-
         // Driver creation can fail with "Cannot start the driver service on http://localhost:56686/" exceptions if the
         // machine is under load. Retrying it here so not the whole test needs to be re-run.
         const int maxTryCount = 3;
@@ -79,12 +85,14 @@ Task<T> FromAsync<T>(Func<BrowserConfiguration, TimeSpan, Task<T>> factory)
         {
             try
             {
+                var pageLoadTimeout = timeoutConfiguration.PageLoadTimeout;
+
                 return browserConfiguration.Browser switch
                 {
-                    Browser.Chrome => await FromAsync(WebDriverFactory.CreateChromeDriverAsync),
-                    Browser.Edge => await FromAsync(WebDriverFactory.CreateEdgeDriverAsync),
-                    Browser.Firefox => await FromAsync(WebDriverFactory.CreateFirefoxDriverAsync),
-                    Browser.InternetExplorer => await FromAsync(WebDriverFactory.CreateInternetExplorerDriverAsync),
+                    Browser.Chrome => await WebDriverFactory.CreateChromeDriverAsync(browserConfiguration, pageLoadTimeout),
+                    Browser.Edge => await WebDriverFactory.CreateEdgeDriverAsync(browserConfiguration, pageLoadTimeout),
+                    Browser.Firefox => await WebDriverFactory.CreateFirefoxDriverAsync(browserConfiguration, pageLoadTimeout),
+                    Browser.InternetExplorer => await WebDriverFactory.CreateInternetExplorerDriverAsync(browserConfiguration, pageLoadTimeout),
                     _ => throw new InvalidOperationException($"Unknown browser: {browserConfiguration.Browser}."),
                 };
             }

Lombiq.Tests.UI/Services/AtataScope.cs

@@ -13,7 +13,18 @@ public sealed class AtataScope : IDisposable
     private Uri _baseUri;
 
     public AtataContext AtataContext { get; }
-    public IWebDriver Driver => AtataContext.Driver;
+
+    public IWebDriver Driver
+    {
+        get
+        {
+            var driver = AtataContext.Driver;
+            IsBrowserRunning = driver != null;
+            return driver;
+        }
+    }
+
+    public bool IsBrowserRunning { get; private set; }
 
     public Uri BaseUri
     {

Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs

@@ -6,6 +6,7 @@
 namespace Lombiq.Tests.UI.Services;
 
 public delegate Task BeforeSetupHandler(OrchardCoreUITestExecutorConfiguration configuration);
+public delegate Task AfterSetupHandler(OrchardCoreUITestExecutorConfiguration configuration);
 
 /// <summary>
 /// Configuration for the initial setup of an Orchard Core app.
@@ -35,4 +36,5 @@ public class OrchardCoreSetupConfiguration
         Path.Combine(DirectoryPaths.Temp, DirectoryPaths.SetupSnapshot);
 
     public BeforeSetupHandler BeforeSetup { get; set; }
+    public AfterSetupHandler AfterSetup { get; set; }
 }

Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs

@@ -20,6 +20,12 @@ public enum Browser
     Edge,
     Firefox,
     InternetExplorer,
+
+    /// <summary>
+    /// No browser will be used. Useful for testing things that don't require a browser, like API endpoints or running
+    /// security scans.
+    /// </summary>
+    None,
 }
 
 public class OrchardCoreUITestExecutorConfiguration

Lombiq.Tests.UI/Services/SynchronizingWebApplicationSnapshotManager.cs

@@ -7,7 +7,7 @@
 
 namespace Lombiq.Tests.UI.Services;
 
-public delegate Task<(UITestContext Context, Uri ResultUri)> AppInitializer();
+public delegate Task<(UITestContext Context, Uri TestStartUri)> AppInitializer();
 
 /// <summary>
 /// Service for transparently running operations on a web application and snapshotting them just a single time, so the
@@ -25,7 +25,7 @@ public class SynchronizingWebApplicationSnapshotManager
     private readonly SemaphoreSlim _semaphore = new(1, 1);
     private readonly string _snapshotDirectoryPath;
 
-    private Uri _resultUri;
+    private Uri _testStartUri;
     private bool _snapshotCreated;
 
     public SynchronizingWebApplicationSnapshotManager(string snapshotDirectoryPath) => _snapshotDirectoryPath = snapshotDirectoryPath;
@@ -37,7 +37,7 @@ public async Task<Uri> RunOperationAndSnapshotIfNewAsync(AppInitializer appIniti
         await _semaphore.WaitAsync();
         try
         {
-            if (_snapshotCreated) return _resultUri;
+            if (_snapshotCreated) return _testStartUri;
 
             DebugHelper.WriteLineTimestamped("Creating snapshot.");
 
@@ -53,7 +53,7 @@ public async Task<Uri> RunOperationAndSnapshotIfNewAsync(AppInitializer appIniti
             // At the end so if any exception happens above then it won' be mistakenly set to true.
             _snapshotCreated = true;
 
-            return _resultUri ??= result.ResultUri;
+            return _testStartUri ??= result.TestStartUri;
         }
         finally
         {