Merge pull request #109 from Lombiq/task/monkey-testing

Monkey testing

.gitignore

@@ -4,3 +4,5 @@ bin/
 *.user
 Localization/
 wwwroot
+node_modules/
+package-lock.json

Lombiq.Tests.UI.Samples/Readme.md

@@ -24,6 +24,7 @@ For general details about and on using the Toolbox see the [root Readme](../Read
 - [Using SQL Server](Tests/SqlServerTests.cs)
 - [Using Azure Blob Storage](Tests/AzureBlobStorageTests.cs)
 - [Error handling](Tests/ErrorHandlingtests.cs)
+- [Monkey tests](Tests/MonkeyTests.cs)
 
 
 ## Adding new tutorials

Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs

@@ -44,3 +44,4 @@ public Task ErrorOnLoadedPageShouldHaltTest(Browser browser) =>
 }
 
 // END OF TRAINING SECTION: Error handling.
+// NEXT STATION: Head over to Tests/MonkeyTests.cs.

Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs

@@ -0,0 +1,91 @@
+using Lombiq.Tests.UI.Attributes;
+using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.MonkeyTesting;
+using Lombiq.Tests.UI.Services;
+using System;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples.Tests
+{
+    // It's possible to execute monkey tests that walk through site pages and do random interactions with pages, like
+    // click, scrolling, form filling, etc. Such random actions can uncover bugs that are otherwise difficult to find.
+    // Use such tests plug holes in your test suite which are not covered by explicit tests.
+    public class MonkeyTests : UITestBase
+    {
+        // Monkey testing has its own configuration too. Check out the docs of the options too.
+        private readonly MonkeyTestingOptions _monkeyTestingOptions = new()
+        {
+            PageTestTime = TimeSpan.FromSeconds(10),
+            BaseRandomSeed = 1234,
+        };
+
+        public MonkeyTests(ITestOutputHelper testOutputHelper)
+            : base(testOutputHelper)
+        {
+        }
+
+        // The basic idea is that you unleash monkey testing on specific pages or sections of the site, like a contact
+        // form or the content management UI.
+        // First, we test a single page.
+        [Theory, Chrome]
+        public Task TestCurrentPageAsMonkeyShouldWorkWithConfiguredRandomSeed(Browser browser) =>
+            ExecuteTestAfterSetupAsync(
+                context =>
+                {
+                    // Note how we define the starting point of the test as the homepage.
+                    context.GoToHomePage();
+                    // The specified random see gives you the option to reproduce the random interactions. Otherwise
+                    // it would be calculated from MonkeyTestingOptions.BaseRandomSeed.
+                    context.TestCurrentPageAsMonkey(_monkeyTestingOptions, 12345);
+                },
+                browser);
+
+        // Recursive testing will just continue testing following the configured rules until it runs out time or new
+        // pages.
+        [Theory, Chrome]
+        public Task TestCurrentPageAsMonkeyRecursivelyShouldWorkWithAnonymousUser(Browser browser) =>
+            ExecuteTestAfterSetupAsync(
+                context =>
+                {
+                    context.GoToHomePage();
+                    context.TestCurrentPageAsMonkeyRecursively(_monkeyTestingOptions);
+                },
+                browser);
+
+        // Let's test with an authenticated user too.
+        [Theory, Chrome]
+        public Task TestCurrentPageAsMonkeyRecursivelyShouldWorkWithAdminUser(Browser browser) =>
+            ExecuteTestAfterSetupAsync(
+                context =>
+                {
+                    // Monkey tests needn't all start from the homepage. This one starts from the Orchard admin
+                    // dashboard.
+                    context.SignInDirectlyAndGoToDashboard();
+                    context.TestCurrentPageAsMonkeyRecursively(_monkeyTestingOptions);
+                },
+                browser);
+
+        // Let's just test the background tasks management admin area.
+        [Theory, Chrome]
+        public Task TestAdminBackgroundTasksAsMonkeyRecursivelyShouldWorkWithAdminUser(Browser browser) =>
+            ExecuteTestAfterSetupAsync(
+                context =>
+                {
+                    // You can fence monkey testing with URL filters: Monkey testing will only be executed if the
+                    // current URL matches. This way, you can restrict monkey testing to just sections of the site. You
+                    // can also use such fencing to have multiple monkey testing methods in multiple test classes, thus
+                    // running them in parallel.
+                    _monkeyTestingOptions.UrlFilters.Add(new StartsWithMonkeyTestingUrlFilter("/Admin/BackgroundTasks"));
+                    // You could also configure the same thing with regex:
+                    ////_monkeyTestingOptions.UrlFilters.Add(new MatchesRegexMonkeyTestingUrlFilter(@"\/Admin\/BackgroundTasks"));
+
+                    context.SignInDirectlyAndGoToRelativeUrl("/Admin/BackgroundTasks");
+                    context.TestCurrentPageAsMonkeyRecursively(_monkeyTestingOptions);
+                },
+                browser);
+    }
+}
+
+// END OF TRAINING SECTION: Monkey tests.

Lombiq.Tests.UI/Docs/CreatingTests.md

@@ -8,8 +8,10 @@ Reference `Lombiq.Tests.UI` from your test project, and add a reference to the `
 
 For a sample test project see [`Lombiq.Tests.UI.Samples`](../../Lombiq.Tests.UI.Samples/Readme.md).
 
-We also recommend always running the suite of tests for checking that all the basic Orchard Core features work, like login, registration, and content management. Use `context.TestBasicOrchardFeatures()` to run all such tests but see the other, more granular tests too. This is also demonstrated in `Lombiq.Tests.UI.Samples` and in [this video](https://www.youtube.com/watch?v=jmhq63sRZrI).
+We also recommend always running some highly automated tests that need very little configuration:
 
+- The suite of tests for checking that all the basic Orchard Core features work, like login, registration, and content management. Use `context.TestBasicOrchardFeatures()` to run all such tests but see the other, more granular tests too. This is also demonstrated in `Lombiq.Tests.UI.Samples` and in [this video](https://www.youtube.com/watch?v=jmhq63sRZrI).
+- [Monkey tests](https://en.wikipedia.org/wiki/Monkey_testing) can also be useful. Use `context.TestCurrentPageAsMonkeyRecursively()` to run a monkey testing process, which walks through site pages and does random interactions with pages, like clicking, scrolling, form filling, etc. It's recommended to have at least 3 monkey tests that execute with different user states: As an admin, as a regular registered user and as an anonymous user. The admin test can start execution on admin dashboard page, while other tests can start on home page.
 
 ## Steps for creating a test class
 

Lombiq.Tests.UI/Docs/Tools.md

@@ -12,3 +12,4 @@
 - There are multiple recording tools available for Selenium but the "official" one which works pretty well is [Selenium IDE](https://www.selenium.dev/selenium-ide/) (which is a Chrome/Firefox extension). To fine-tune XPath queries and CSS selectors and also to record tests check out [ChroPath](https://chrome.google.com/webstore/detail/chropath/ljngjbnaijcbncmcnjfhigebomdlkcjo/) (the [Xpath cheatsheet](https://devhints.io/xpath) is a great resource too, and [XmlToolBox](https://xmltoolbox.appspot.com/xpath_generator.html) can help you with quick XPath queries).
 - Accessibility checking can be done with [axe](https://github.com/dequelabs/axe-core) via [Selenium.Axe for .NET](https://github.com/TroyWalshProf/SeleniumAxeDotnet).
 - HTML markup validation can be done with [html-validate](https://gitlab.com/html-validate/html-validate) via [Atata.HtmlValidation](https://github.com/atata-framework/atata-htmlvalidation).
+- Monkey testing is implemented using [Gremlins.js](https://github.com/marmelab/gremlins.js/) library.

Lombiq.Tests.UI/Docs/Troubleshooting.md

@@ -54,3 +54,10 @@
 
     EXEC sp_executesql @sql 
     ```
+
+
+## Monkey testing
+
+- Errors uncovered by monkey testing functionality can be reproduced locally by executing the same test with the same random seed by setting `MonkeyTestingOptions.BaseRandomSeed` with the value the test failed. If `BaseRandomSeed` is generated then you can see it in the log; if you specified it then nothing else to do. The last monkey testing interaction is logged.
+- If you want to test the failed page granularly, you can write a test that navigates to that page and executes `context.TestCurrentPageAsMonkey(_monkeyTestingOptions, 12345);`, where `12345` is the random seed number that can be found in a failed test log.
+- It is also possible to set a larger time value to the `MonkeyTestingOptions.GremlinsAttackDelay` property in order to make gremlin interaction slower, thus allowing you to watch what's happening.

Lombiq.Tests.UI/EmbeddedResourceProvider.cs

@@ -0,0 +1,18 @@
+using System.IO;
+using System.Text;
+
+namespace Lombiq.Tests.UI
+{
+    internal static class EmbeddedResourceProvider
+    {
+        internal static string ReadEmbeddedFile(string fileName)
+        {
+            var assembly = typeof(EmbeddedResourceProvider).Assembly;
+            var resourceStream = assembly.GetManifestResourceStream($"{assembly.GetName().Name}.Resources.{fileName}");
+
+            using var reader = new StreamReader(resourceStream, Encoding.UTF8);
+
+            return reader.ReadToEnd();
+        }
+    }
+}

Lombiq.Tests.UI/Extensions/BasicOrchardFeaturesTestingUITestContextExtensions.cs

@@ -140,7 +140,7 @@ public static UITestContext TestLogin(
                 () =>
                 {
                     context.GoToLoginPage()
-                        .LogInWith(userName, password)
+                        .LogInWith(context, userName, password)
                         .ShouldLeaveLoginPage();
 
                     context.GetCurrentUserName().ShouldBe(userName);
@@ -168,7 +168,7 @@ public static UITestContext TestLoginWithInvalidData(
                     context.SignOutDirectly();
 
                     context.GoToLoginPage()
-                        .LogInWith(userName, password)
+                        .LogInWith(context, userName, password)
                         .ShouldStayOnLoginPage()
                         .ValidationSummaryErrors.Should.Not.BeEmpty();
 
@@ -193,6 +193,8 @@ public static UITestContext TestLogout(this UITestContext context) =>
                         .TopNavbar.Account.LogOff.Click()
                         .ShouldLeaveAdminPage();
 
+                    context.TriggerAfterPageChangeEventAsync().Wait();
+
                     context.GetCurrentUserName().ShouldBeNullOrEmpty();
                 });
 
@@ -220,14 +222,14 @@ public static UITestContext TestRegistration(this UITestContext context, UserReg
                     context.GoToLoginPage()
                         .RegisterAsNewUser.Should.BeVisible()
                         .RegisterAsNewUser.ClickAndGo()
-                            .RegisterWith(parameters)
+                            .RegisterWith(context, parameters)
                             .ShouldLeaveRegistrationPage();
 
                     context.GetCurrentUserName().ShouldBe(parameters.UserName);
                     context.SignOutDirectly();
 
-                    context.GoToLoginPage()
-                        .LogInWith(parameters.UserName, parameters.Password);
+                    context.GoToLoginPage().LogInWith(context, parameters.UserName, parameters.Password);
+                    context.TriggerAfterPageChangeEventAsync().Wait();
                     context.GetCurrentUserName().ShouldBe(parameters.UserName);
                     context.SignOutDirectly();
                 });
@@ -259,7 +261,7 @@ public static UITestContext TestRegistrationWithInvalidData(this UITestContext c
                 "Test registration with invalid data",
                 () => context
                     .GoToRegistrationPage()
-                    .RegisterWith(parameters)
+                    .RegisterWith(context, parameters)
                     .ShouldStayOnRegistrationPage()
                     .ValidationMessages.Should.Not.BeEmpty());
         }
@@ -288,7 +290,7 @@ public static UITestContext TestRegistrationWithAlreadyRegisteredEmail(
                 "Test registration with already registered email",
                 () => context
                     .GoToRegistrationPage()
-                    .RegisterWith(parameters)
+                    .RegisterWith(context, parameters)
                     .ShouldStayOnRegistrationPage()
                     .ValidationMessages[page => page.Email].Should.BeVisible());
         }
@@ -323,6 +325,8 @@ public static UITestContext TestContentOperations(this UITestContext context, st
                         .AlertMessages.Should.Contain(message => message.IsSuccess)
                         .Items[item => item.Title == pageTitle].View.Click();
 
+                    context.TriggerAfterPageChangeEventAsync().Wait();
+
                     context.Scope.AtataContext.Go.ToNextWindow(new OrdinaryPage(pageTitle))
                         .AggregateAssert(page => page
                             .PageTitle.Should.Contain(pageTitle)

Lombiq.Tests.UI/Extensions/EventsOrchardCoreUITestExecutorConfigurationExtensions.cs

@@ -1,8 +1,5 @@
-using Lombiq.Tests.UI.Exceptions;
 using Lombiq.Tests.UI.Models;
 using Lombiq.Tests.UI.Services;
-using OpenQA.Selenium;
-using System;
 using System.Threading.Tasks;
 
 namespace Lombiq.Tests.UI.Extensions
@@ -15,7 +12,7 @@ public static void SetUpEvents(this OrchardCoreUITestExecutorConfiguration confi
 
             PageNavigationState navigationState = null;
 
-            configuration.Events.AfterNavigation += (context, _) => OnEventsAfterNavigationAsync(context);
+            configuration.Events.AfterNavigation += (context, _) => context.TriggerAfterPageChangeEventAsync();
 
             configuration.Events.BeforeClick += (context, _) =>
             {
@@ -25,40 +22,8 @@ public static void SetUpEvents(this OrchardCoreUITestExecutorConfiguration confi
 
             configuration.Events.AfterClick += (context, _) =>
                 navigationState.CheckIfNavigationHasOccurred()
-                    ? OnEventsAfterNavigationAsync(context)
+                    ? context.TriggerAfterPageChangeEventAsync()
                     : Task.CompletedTask;
         }
-
-        private static bool IsNoAlert(UITestContext context)
-        {
-            // If there's an alert (which can happen mostly after a click but also after navigating) then all other
-            // driver operations, even retrieving the current URL, will throw an UnhandledAlertException. Thus we
-            // need to check if an alert is present and that's only possible by catching exceptions.
-            try
-            {
-                context.Driver.SwitchTo().Alert();
-                return false;
-            }
-            catch (NoAlertPresentException)
-            {
-                return true;
-            }
-        }
-
-        private static async Task OnEventsAfterNavigationAsync(UITestContext context)
-        {
-            if (IsNoAlert(context) &&
-                context.Configuration.Events.AfterPageChange is { } afterPageChange)
-            {
-                try
-                {
-                    await afterPageChange.Invoke(context);
-                }
-                catch (Exception exception)
-                {
-                    throw new PageChangeAssertionException(context, exception);
-                }
-            }
-        }
     }
 }