From 34558ec0bc7174706cdbf43792de4fc71e60699c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 15 Sep 2026 05:22:19 +0100 Subject: [PATCH 1/7] TEST: Move the 51Did browser acceptance tests here from the cloud, as they were The fifteen browser acceptance tests for the 51Did user prompt work were written into the cloud repository at Did/Tests/Selenium.Context.Integration/Browser51Did on the branch feature/create-last-f. They prove client script behaviour that every implementation shares, so they belong in this suite, where every implementation's example can be pointed at them. This commit is the files as they were, with only what they need to build here: the namespace moved under FiftyOne.Pipeline.Cloud.SeleniumTests, a nullable context per file because this project does not enable one, and the published FiftyOne.Did parser the tests read an identifier with. Every class keeps TestCategory("Browser51Did"). Configuration, the suite's own furniture and the implementation under test follow in the next commits, so this one can be compared line for line with the cloud copy. --- Browser51Did/Browser51DidTestBase.cs | 153 +++++ .../ConsentPlatformAcceptanceTests.cs | 182 +++++ Browser51Did/Harness.cs | 632 ++++++++++++++++++ Browser51Did/IsGdprAcceptanceTests.cs | 125 ++++ Browser51Did/PageServer.cs | 193 ++++++ Browser51Did/Pages.cs | 324 +++++++++ Browser51Did/PlatformAcceptanceTests.cs | 556 +++++++++++++++ Browser51Did/PlatformAddsClientScriptTests.cs | 234 +++++++ Browser51Did/TcString.cs | 76 +++ Browser51Did/Visitor.cs | 588 ++++++++++++++++ SeleniumApiTests.csproj | 9 + 11 files changed, 3072 insertions(+) create mode 100644 Browser51Did/Browser51DidTestBase.cs create mode 100644 Browser51Did/ConsentPlatformAcceptanceTests.cs create mode 100644 Browser51Did/Harness.cs create mode 100644 Browser51Did/IsGdprAcceptanceTests.cs create mode 100644 Browser51Did/PageServer.cs create mode 100644 Browser51Did/Pages.cs create mode 100644 Browser51Did/PlatformAcceptanceTests.cs create mode 100644 Browser51Did/PlatformAddsClientScriptTests.cs create mode 100644 Browser51Did/TcString.cs create mode 100644 Browser51Did/Visitor.cs diff --git a/Browser51Did/Browser51DidTestBase.cs b/Browser51Did/Browser51DidTestBase.cs new file mode 100644 index 0000000..91389da --- /dev/null +++ b/Browser51Did/Browser51DidTestBase.cs @@ -0,0 +1,153 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using FiftyOne.Did.Model; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// What every browser acceptance test class shares, being the guard, the +/// skip when the harness is not configured, and the reading of a 51Did. +/// +/// The guard runs once for each class from the set up, not from inside a +/// test, so that a container built against the old client script fails +/// every test in the class rather than letting one of them pass for the +/// wrong reason. +/// +/// +public abstract class Browser51DidTestBase +{ + /// + /// The browsers these tests are written for. Both are run where the + /// behaviour under test differs between them, which is anything the + /// shared choice travels on. + /// + protected const string Chrome = "Chrome"; + + /// The second browser, for the same reason. + protected const string Firefox = "Firefox"; + + /// + /// Fails every test in the class unless the client script served by + /// the endpoint under test is the new one. See + /// . + /// + [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] + public static void GuardTheClientScript(TestContext context) + { + if (Harness.Configured == false) + { + // Nothing to guard. Each test reports inconclusive below and + // says which variables are missing. + return; + } + Harness.RequireTheNewClientScript(); + } + + [TestInitialize] + public void RequireHarness() + { + if (Harness.Configured == false) + { + Assert.Inconclusive( + "The browser acceptance harness is not configured. Set " + + "FIFTYONE_CONTEXT_SELENIUM_BASEURL and " + + "FIFTYONE_CONTEXT_SELENIUM_RESOURCE, or run ci/test.ps1 " + + "with -Browser51Did $true, which starts the container and " + + "sets them. See README.md."); + } + } + + /// + /// A browser looking at a website served for the length of this test. + /// + protected static Visitor NewVisitor( + string browser, + PageServer server, + IReadOnlyDictionary? preferences = null) + => new( + browser == Firefox + ? Harness.NewFirefox(preferences) + : Harness.NewChrome(preferences), + server, + browser); + + /// + /// Refuses to go on where the cloud will not hold a choice for this + /// resource key, because the test would then be failing on the harness + /// rather than on the code. The service is asked rather than a flag + /// being read, so the reason reported is the service's own. + /// + protected static void RequireSharing() => Harness.RequireSharedStore(); + + /// + /// Refuses to go on where this resource key cannot create an + /// identifier for a standard or personalized answer, which needs a + /// licence carrying the CloudV5FODiD product. See + /// for why a throwaway record cannot + /// have one. + /// + protected static void RequireMarketingIdentifiers() + => Harness.RequireMarketingIdentifiers(); + + #region Reading a 51Did + + /// + /// Parses an identifier the page is holding, failing with what was + /// wrong when it is not one. + /// + protected static FodId Parse(string identifier, string what) + { + Assert.IsFalse( + string.IsNullOrWhiteSpace(identifier), + $"{what}: the page holds no identifier at all."); + Assert.IsTrue( + FodId.TryParse(identifier, out var parsed, out var status), + $"{what}: '{identifier}' is not a 51Did ({status})."); + Assert.IsNotNull( + parsed, + $"{what}: '{identifier}' parsed to nothing ({status})."); + return parsed!; + } + + /// + /// The identifier says the usage was stated by the caller, which is + /// what the preference platform does, so the signal source is direct. + /// + protected static void AssertDirect(string identifier, string what) + { + var parsed = Parse(identifier, what); + Assert.IsFalse( + parsed.UsageFromConsent, + $"{what}: the usage should have been stated directly, so the " + + "flag saying it was decoded from a framework string must be " + + $"clear. The identifier reports usage {parsed.Usage}."); + } + + /// + /// The identifier says the usage was decoded from a framework string, + /// which is what a consent platform page produces. + /// + protected static void AssertFromConsentString( + string identifier, string what, Usage expected) + { + var parsed = Parse(identifier, what); + Assert.IsTrue( + parsed.UsageFromConsent, + $"{what}: the usage came from a framework string, so the flag " + + "recording that must be set. The identifier reports usage " + + $"{parsed.Usage}."); + Assert.AreEqual( + expected, + parsed.Usage, + $"{what}: the framework string granted {expected}."); + } + + /// The usage an identifier carries. + protected static Usage UsageOf(string identifier, string what) + => Parse(identifier, what).Usage; + + #endregion +} diff --git a/Browser51Did/ConsentPlatformAcceptanceTests.cs b/Browser51Did/ConsentPlatformAcceptanceTests.cs new file mode 100644 index 0000000..fd5e3e9 --- /dev/null +++ b/Browser51Did/ConsentPlatformAcceptanceTests.cs @@ -0,0 +1,182 @@ +#nullable enable + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using FiftyOne.Did.Model; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// A page with a consent platform on it and no preference platform, which +/// is the other half of demonstration 2 and the whole of demonstration 3's +/// second case, being the identifier whose usage was decoded from a +/// framework string rather than stated. +/// +/// The consent platform here is a stub, and that is the contract rather +/// than a gap. The client script uses only what the framework's +/// specification requires of every platform, being ping, +/// addEventListener and a callback carrying tcString and eventStatus, so a +/// stub honouring those is what every product has to do, and a product +/// that breaks it is the product's defect. The two never share a page, so +/// nothing here also carries the preference platform. +/// +/// +[TestClass, TestCategory("Browser51Did")] +public class ConsentPlatformAcceptanceTests : Browser51DidTestBase +{ + /// + /// The publisher writes nothing but the script tag and the stub the + /// framework's own specification already tells them to write. The + /// answer arrives after the script's first round, which is the common + /// case, and the identifier that comes back records that the usage was + /// decoded rather than stated. + /// + [TestMethod] + public void ConsentPlatformOnly_SecondRequestCarriesTheString() + { + using var server = new PageServer(); + RequireMarketingIdentifiers(); + server.Put("/", Pages.Page( + "Consent platform only", + Pages.ConsentPlatform(TcString.Personalized()), + Pages.ClientScriptTag())); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForClientRounds(1); + Assert.IsTrue( + visitor.HasTheNewClientScript(), + "the page is running a client script with no user prompt block " + + "in it, so nothing here would be proving the new behaviour."); + + var first = visitor.ClientRequests()[0]; + Assert.IsNull( + first.Form("id.usage"), + "no answer has been given yet, and a page with a consent " + + "platform on it means unknown rather than none, so nothing " + + $"may be stated. The body was: {first.Body}"); + Assert.AreEqual( + "", + visitor.Identifier(), + "no answer means no identifier."); + + // The visitor answers the consent platform, which delivers the + // string to whoever registered. The test decides when, so the + // ordering is asserted rather than timed. + var listeners = visitor.Script( + "return window.__51dCmp.deliver();"); + Assert.IsTrue( + listeners > 0, + "the client script must have registered a listener with the " + + "consent platform. Nothing was registered, so the answer " + + "reached nobody. The console said: " + + string.Join(" | ", visitor.Console())); + + Harness.Until( + () => visitor.ClientRequests().Count(r => r.Done) >= 2, + "the client script asked again once the answer arrived. The " + + "console said: " + string.Join(" | ", visitor.Console())); + + var second = visitor.ClientRequests().Last(); + Assert.IsNotNull( + second.Form("tcstring"), + "the framework string the platform delivered must reach the " + + $"cloud. The body was: {second.Body}"); + Assert.IsNull( + second.Form("id.usage"), + "the client script must not decide the usage itself. A string " + + "goes as a string and the server decodes it, which is what " + + "makes the flag mean something. The body was: " + + second.Body); + Assert.AreEqual( + 1, + second.FormCount("tcstring"), + "the string must be sent once, because the server takes the " + + "first value of a repeated key and warns. The body was: " + + second.Body); + + Harness.Until( + () => visitor.Identifier() != "", + "an identifier came back once the answer had been decoded. The " + + "console said: " + string.Join(" | ", visitor.Console())); + Assert.IsTrue( + second.Response.Contains("fodid", StringComparison.Ordinal), + "the response to the request carrying the string must carry the " + + "identifier section."); + AssertFromConsentString( + visitor.Identifier(), + "the usage was decoded from the framework string", + Usage.Personalized); + } + + /// + /// Demonstration 8. A page with nowhere to get an answer from creates + /// nothing, says so once, and stores a record that carries no answer, + /// so a later page view with an answer is a different input and asks + /// again rather than reusing this one. + /// + [TestMethod] + public void NoPlatformAtAll_NoIdentifierAndOneWarning() + { + using var server = new PageServer(); + server.Put("/", Pages.Page( + "No platform", Pages.ClientScriptTag())); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForClientRounds(1); + Assert.IsTrue( + visitor.HasTheNewClientScript(), + "the page is running a client script with no user prompt block " + + "in it, so it would have nothing to warn about."); + + var requests = visitor.ClientRequests(); + Assert.AreEqual( + 1, + requests.Count, + "with nowhere to get an answer from there is nothing to come " + + "back for, so one request is all there should be. The " + + "requests were: " + + string.Join(" || ", requests.Select(r => r.ToString()))); + Assert.IsNull( + requests[0].Form("id.usage"), + $"nobody was asked. The body was: {requests[0].Body}"); + Assert.IsNull( + requests[0].Form("tcstring"), + $"there is no platform to ask. The body was: {requests[0].Body}"); + Assert.AreEqual( + "", + visitor.Identifier(), + "a page where nobody was asked produces no identifier, which is " + + "the intended outcome and not a fault."); + + var record = visitor.CacheRecord(); + Assert.IsNotNull( + record, + "the client script stores the inputs of the request it made, so " + + "that a later page view with a different input asks again. " + + "Nothing was stored."); + Assert.IsFalse( + record!.Contains("id.usage", StringComparison.Ordinal), + "the stored record must carry no answer, or a page view that " + + "does have one would look the same as this and be served " + + $"from the cache. The record was: {record}"); + + var warnings = visitor.ConsoleMatching(Harness.NoPlatformMessage); + Assert.AreEqual( + 1, + warnings.Count, + "the warning that there is no platform on the page is logged " + + "once for the page view, not once per round. The console " + + "said: " + string.Join(" | ", visitor.Console())); + foreach (var empty in new[] { "undefined", "null" }) + { + Assert.IsFalse( + warnings[0].Contains(empty, StringComparison.OrdinalIgnoreCase), + "the warning names nothing it could not find a value for, " + + $"so '{empty}' must not appear in it. It said: " + + warnings[0]); + } + } +} diff --git a/Browser51Did/Harness.cs b/Browser51Did/Harness.cs new file mode 100644 index 0000000..5c35558 --- /dev/null +++ b/Browser51Did/Harness.cs @@ -0,0 +1,632 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Firefox; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// Everything the browser acceptance tests share, being where the cloud +/// is, which resource key to use, the two site names a cross site test +/// needs, the browsers, and the guard that stops the whole class running +/// against the old client script. +/// +/// The harness is the same one CrossBrowserContextTests uses, being +/// the container that terminates TLS itself, started by +/// ci/test.ps1 with a throwaway resource key and a throwaway +/// context secret. These tests reuse its environment variables rather +/// than adding a second set. +/// +/// +public static class Harness +{ + /// + /// The cloud instance serving TLS itself, for example + /// https://localhost:8081. Shared with the cross browser context + /// tests, so one container serves both. + /// + public static readonly string? BaseUrl = + Environment.GetEnvironmentVariable( + "FIFTYONE_CONTEXT_SELENIUM_BASEURL"); + + /// + /// The resource key the pages use. + /// + /// It is not the throwaway one the cross browser context tests use. + /// Creating an identifier for a standard or personalized answer needs + /// a licence carrying the CloudV5FODiD product, which + /// DidOnPremiseEngine.TryResolveLicenseId scans the customer's + /// licence keys for, and refusing without one is deliberate. A + /// throwaway record cannot have that product, because the product + /// lives inside a real signed licence key rather than being a name a + /// record can claim, which is why the context tests ask for the + /// non-marketing usage. Everything the acceptance tests are for is a + /// marketing usage, so they use the 51Did entitled resource key that + /// is already committed in + /// host/FiftyOne.Pipeline.Cloud.Tests.Common/testConfig.json + /// as fodid_resource_key and that ci/test.ps1 already uses for + /// the identifier surface check. That customer's record also carries + /// a licence key, which is what the shared store needs. + /// + /// + /// It falls back to the context harness key so that a developer who + /// sets only the two context variables gets a run that says plainly + /// what it could not do rather than one that will not start. + /// + /// + public static readonly string? Resource = + FirstSet( + "FIFTYONE_BROWSER51DID_RESOURCE", + "FIFTYONE_CONTEXT_SELENIUM_RESOURCE"); + + private static string? FirstSet(params string[] names) + { + foreach (var name in names) + { + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrEmpty(value) == false) + { + return value; + } + } + return null; + } + + /// + /// Whether the harness is configured at all. Unset means a developer + /// ran the project on its own, so the tests report inconclusive rather + /// than failing, exactly as the context tests do. + /// + public static bool Configured => + string.IsNullOrEmpty(BaseUrl) == false + && string.IsNullOrEmpty(Resource) == false; + + /// + /// The first publisher site. It is a name and not localhost, because + /// the cloud is on localhost and a page there would be the same site + /// as the cloud, which would make the shared cookie a first party one + /// and prove nothing. + /// + public const string SiteA = "site-a.localtest"; + + /// + /// The second publisher site, for the cross site case. A different + /// registrable name from , so a browser treats the + /// two as different sites rather than as one. + /// + /// The top level domain is one nothing owns, so neither name can ever + /// resolve on the network and neither can be reached by accident. The + /// browsers are told to resolve both to this machine, and the + /// development certificate used when the service is run outside a + /// container names both, so the same two names work whichever way the + /// service under test was started. + /// + /// + public const string SiteB = "site-b.localtest"; + + /// + /// How long any wait in these tests is given before it is called a + /// failure. Nothing is asserted on a clock, so this is only the point + /// at which a test stops waiting for something that is never going to + /// happen. + /// + public static readonly TimeSpan Patience = TimeSpan.FromSeconds(30); + + #region Console messages the tests look for + + /// + /// Text that exists only in the client script's new user prompt block, + /// being the message it logs once a page view has reached the server's + /// maximum number of rounds. The guard below refuses to let any test + /// in this namespace run unless the script served by the endpoint under + /// test carries it, which is what makes a green run mean something. + /// + /// Settled by the template work package. If that wording changes, this + /// is the one place to change it, and the report for that package + /// carries the string in force. + /// + /// + public const string IterationLimitMessage = "51Degrees: the maximum of"; + + /// + /// Text that exists only inside the template's user prompt section, + /// being the name the preference platform puts its own surface under. + /// The iteration limit message above sits outside that section, so it + /// says the template is the new one and says nothing about whether the + /// block was rendered. This says the block is there. + /// + public const string UserPromptBlockMarker = "__51d_pmp"; + + /// + /// The warning the client script logs when a page carries no place to + /// get an answer from, so no identifier can be created. Read off the + /// script the endpoint served on 15 September 2026, where the sentence + /// is "51Degrees: no preference platform was found on this page. A + /// platform's stub must precede this script. No 51Did will be created + /// until a platform answers." Matched as a substring, so the rest of + /// the sentence may change, and the test using it also asserts that no + /// value was printed beside it. + /// + public const string NoPlatformMessage = + "no preference platform was found"; + + /// + /// The warning the client script logs when a second copy of itself is + /// loaded onto one page under the same object name, where the sentence + /// is "51Degrees: fod already exists on this page. Loading the script + /// twice replaces it. Load it once and call fod.refresh() to update." + /// The object's name is in it, so only the part that is the same + /// whatever the object is called is matched. Two tests assert it was + /// NOT logged, which is how they say that nothing added a second copy. + /// + public const string SecondInstanceWarning = + "already exists on this page"; + + /// + /// What the preference platform says when it finds no client script + /// object on the page. Read from the platform's own source, where the + /// sentence is "There is no client script object named '{name}' on + /// this page, so the client script is being added from the cloud that + /// served this one." + /// + public const string NoClientScriptMessage = + "no client script object named"; + + /// + /// And what it says when it goes on to add one, which is the half a + /// publisher who left the tag out has to see. + /// + public const string AddingClientScriptMessage = + "client script is being added"; + + #endregion + + /// + /// The client script for the throwaway resource key, as the page asks + /// for it. + /// + public static string ClientScriptUrl(string? objectName = null) + { + var url = $"{BaseUrl}/api/v4/{Resource}.js"; + return objectName is null + ? url + : $"{url}?fod-js-object-name={objectName}"; + } + + /// The preference platform's loader, as the page asks for it. + public static string PlatformLoaderUrl() => $"{BaseUrl}/api/v4/pmp"; + + /// + /// Server to server, with certificate validation relaxed because the + /// container under test serves a certificate issued for another name. + /// The browsers are told to accept it for the same reason. + /// + private static readonly HttpClient Reader = new( + new HttpClientHandler + { + ServerCertificateCustomValidationCallback = + (message, certificate, chain, errors) => true, + }) + { + Timeout = TimeSpan.FromSeconds(60), + }; + + private static readonly object GuardLock = new(); + private static string? _guardEvidence; + private static string? _guardFailure; + + /// + /// The guard. Fetches the client script from the endpoint the tests are + /// about to drive a browser at and refuses to go on unless the body + /// carries both of the markers below. + /// + /// Two markers, because they say different things. The iteration limit + /// message says the template is the new one. It sits outside the user + /// prompt section, so it is in the rendered script whenever updates + /// are enabled, whether or not the block itself was rendered. The + /// platform's own global name appears only inside the section, so it + /// is what says the block is actually there. A run that had only the + /// first would be testing the new template with the block switched + /// off, which passes nothing it is meant to prove. + /// + /// + /// It runs once per class from the set up rather than inside a test, so + /// no test here can run against the old script and report green. A run + /// against a cloud built on the released 4.5.104 package fails every + /// test in the class with the line below, which says what was served + /// and what was wanted. + /// + /// + public static void RequireTheNewClientScript() + { + lock (GuardLock) + { + if (_guardFailure != null) + { + Assert.Fail(_guardFailure); + } + if (_guardEvidence != null) + { + Console.WriteLine(_guardEvidence); + return; + } + string body; + var url = ClientScriptUrl(); + try + { + body = Reader.GetStringAsync(url).Result; + } + catch (Exception error) + { + _guardFailure = + "The client script could not be fetched from " + + $"{Redacted(url)}, " + + "so there is no way to tell which template the " + + $"container was built from. {error.Message}"; + Assert.Fail(_guardFailure); + return; + } + var evidence = new List(); + foreach (var (marker, says) in RequiredMarkers) + { + var found = body.IndexOf(marker, StringComparison.Ordinal); + if (found < 0) + { + _guardFailure = + "The client script served by " + + $"{Redacted(url)} does not carry " + + $"'{marker}', which is what says {says}. Every " + + "test in this class would be proving the wrong " + + "thing, so none of them runs. Move the " + + "FiftyOne.Pipeline.JavaScriptBuilder pin to the " + + "package built from the new template and build " + + $"the image again. The script was {body.Length} " + + "bytes."; + Assert.Fail(_guardFailure); + return; + } + evidence.Add( + $"'{marker}' at {found}: {Around(body, found)}"); + } + _guardEvidence = + "Client script guard passed. " + + $"{Redacted(url)} served {body.Length} " + + "bytes carrying " + string.Join(" and ", evidence); + Console.WriteLine(_guardEvidence); + } + } + + /// + /// What the served client script has to carry, and what each one + /// proves. See for why one is + /// not enough. + /// + private static readonly (string Marker, string Says)[] RequiredMarkers = + { + (IterationLimitMessage, "the template is the new one"), + (UserPromptBlockMarker, "the user prompt block was rendered"), + }; + + /// + /// The guard's evidence, for a report that has to quote it. Null until + /// has run and passed. + /// + public static string? GuardEvidence => _guardEvidence; + + /// + /// Asks a check until it is happy or until it has asked enough times + /// to say the answer is settled. + /// + /// A service that has just come up answers the first request of a kind + /// before everything behind it is warm, and one cold answer is not a + /// refusal. A real refusal reads the same every time, so this costs a + /// few seconds there and rescues a run that would otherwise skip every + /// test on a first answer nobody would have accepted. + /// + /// + private static string? Settled(Func check) + { + string? refusal = null; + for (var attempt = 0; attempt < 6; attempt++) + { + refusal = check(); + if (refusal == null) + { + return null; + } + Thread.Sleep(2000); + } + return refusal; + } + + private static string? _marketingRefusal; + private static bool _marketingChecked; + + /// + /// Refuses to go on where this resource key cannot create an + /// identifier for a marketing answer, which is what standard and + /// personalized both are. + /// + /// The service is asked rather than a flag being read, so the reason + /// the test reports is the service's own words rather than a guess, + /// and a key that stops carrying the product later says so instead of + /// failing somewhere in a browser. + /// + /// + public static void RequireMarketingIdentifiers() + { + lock (GuardLock) + { + if (_marketingChecked == false) + { + _marketingChecked = true; + _marketingRefusal = Settled(MarketingRefusal); + } + } + if (_marketingRefusal != null) + { + Assert.Inconclusive( + "This resource key cannot create an identifier for a " + + "marketing answer, which is what standard and " + + "personalized both are, so nothing here would be " + + "testing the thing it is for. The service said: " + + _marketingRefusal + + " Point FIFTYONE_BROWSER51DID_RESOURCE at a resource key " + + "whose customer holds a licence carrying the CloudV5FODiD " + + "product, such as fodid_resource_key in " + + "host/FiftyOne.Pipeline.Cloud.Tests.Common/testConfig.json."); + } + } + + /// + /// Asks the service to make a standard identifier and reports why it + /// would not, or null where it did. + /// + private static string? MarketingRefusal() + { + var url = $"{BaseUrl}/api/v4/json?resource={Resource}" + + "&id.usage=standard&values=FODiD.IdProbGlobal"; + string body; + try + { + body = Reader.GetStringAsync(url).Result; + } + catch (Exception error) + { + return $"the service could not be asked. {error.Message}"; + } + JsonElement fodid; + try + { + var root = JsonDocument.Parse(body).RootElement; + if (root.TryGetProperty("fodid", out fodid) == false) + { + return "the response carried no fodid section at all, so " + + "the key is not entitled to the identifier " + + "properties."; + } + } + catch (JsonException) + { + return $"the response was not readable. {Truncate(body)}"; + } + if (fodid.TryGetProperty("idprobglobal", out var value) + && value.ValueKind == JsonValueKind.String) + { + return null; + } + return fodid.TryGetProperty( + "idprobglobalnullreason", out var reason) + ? reason.GetString() + : "no identifier came back and no reason was given."; + } + + private static string? _sharedStoreRefusal; + private static bool _sharedStoreChecked; + + /// + /// Refuses to go on where the shared store will not take a write for + /// this resource key. The store's cookie is named from the customer's + /// licence key, and the controller answers a record carrying none with + /// one refusal, so asking it once is what tells the tests whether a + /// choice can be carried between sites at all. + /// + public static void RequireSharedStore() + { + lock (GuardLock) + { + if (_sharedStoreChecked == false) + { + _sharedStoreChecked = true; + _sharedStoreRefusal = Settled(SharedStoreRefusal); + } + } + if (_sharedStoreRefusal != null) + { + Assert.Inconclusive( + "The cloud will not hold a choice for this resource key, " + + "so a choice cannot be carried between sites and nothing " + + "here would be testing that it is. The service said: " + + _sharedStoreRefusal); + } + } + + /// + /// Offers the shared store a write and reports why it would not take + /// it, or null where it did. Nothing is kept, because the answer's + /// cookie goes to this client rather than to any browser and this + /// client holds none. + /// + private static string? SharedStoreRefusal() + { + try + { + using var request = new HttpRequestMessage( + HttpMethod.Post, $"{BaseUrl}/api/v4/pmp/pref") + { + Content = new FormUrlEncodedContent( + new Dictionary + { + ["resource"] = Resource ?? string.Empty, + ["preference"] = "standard", + ["network"] = NetworkName, + }), + }; + request.Headers.Referrer = new Uri($"http://{SiteA}/"); + using var response = Reader.Send(request); + if (response.IsSuccessStatusCode) + { + return null; + } + var body = response.Content.ReadAsStringAsync().Result; + return $"{(int)response.StatusCode} {Truncate(body)}"; + } + catch (Exception error) + { + return error.Message; + } + } + + /// + /// The group of sites a choice is shared across in these tests. The + /// name is what the cookie holding the choice is derived from, so the + /// pages and the check above have to use the same one. + /// + public const string NetworkName = "Fifty One Network"; + + private static string Truncate(string value) + => value.Length <= 300 ? value : value.Substring(0, 300) + "..."; + + /// + /// A URL with the resource key taken out of it, for anything a test + /// prints. A failure message and the guard's evidence both end up in a + /// run log and in a pull request body, and a resource key is a + /// credential that must never be written down anywhere. + /// + public static string Redacted(string url) + => string.IsNullOrEmpty(Resource) + ? url + : url.Replace(Resource, "", StringComparison.Ordinal); + + /// One line of the script either side of the match. + private static string Around(string body, int at) + { + var from = Math.Max(0, at - 40); + var to = Math.Min(body.Length, at + 120); + return body.Substring(from, to - from) + .Replace("\r", " ", StringComparison.Ordinal) + .Replace("\n", " ", StringComparison.Ordinal); + } + + #region Browsers + + /// + /// Chrome, with both publisher site names pointed at this machine. + /// Nothing is added to the hosts file and nothing has to resolve on + /// the network, which is what makes the cross site case run on any + /// runner. + /// + public static IWebDriver NewChrome( + IReadOnlyDictionary? preferences = null, + params string[] extraArguments) + { + var options = new ChromeOptions(); + options.AddArgument("--headless=new"); + options.AddArgument("--no-sandbox"); + options.AddArgument("--disable-dev-shm-usage"); + options.AddArgument( + "--host-resolver-rules=" + + $"MAP {SiteA} 127.0.0.1,MAP {SiteB} 127.0.0.1"); + options.AcceptInsecureCertificates = true; + foreach (var argument in extraArguments) + { + options.AddArgument(argument); + } + if (preferences != null) + { + foreach (var preference in preferences) + { + options.AddUserProfilePreference( + preference.Key, preference.Value); + } + } + return new ChromeDriver(options); + } + + /// + /// Firefox, with the same two site names pointed at this machine. + /// Firefox has no host resolver rules, so network.dns.localDomains is + /// used, which is the setting that does the same job. + /// + public static IWebDriver NewFirefox( + IReadOnlyDictionary? preferences = null) + { + var options = new FirefoxOptions { AcceptInsecureCertificates = true }; + options.AddArgument("-headless"); + options.SetPreference("network.dns.localDomains", $"{SiteA},{SiteB}"); + // Third party cookies are what the shared choice travels on, so + // the tests state the behaviour they are written for rather than + // inheriting whatever the installed build defaults to. 0 is + // "accept all", and the pair below is Firefox's own total cookie + // protection, which would otherwise keep the cloud's cookie in a + // separate jar per site and hide a working exchange. + options.SetPreference("network.cookie.cookieBehavior", 0); + options.SetPreference( + "privacy.partition.network_state.ocsp_cache", false); + if (preferences != null) + { + foreach (var preference in preferences) + { + switch (preference.Value) + { + case bool flag: + options.SetPreference(preference.Key, flag); + break; + case int number: + options.SetPreference(preference.Key, number); + break; + default: + options.SetPreference( + preference.Key, + Convert.ToString( + preference.Value, + CultureInfo.InvariantCulture)); + break; + } + } + } + return new FirefoxDriver(options); + } + + /// + /// Waits for a condition the page reports, polling rather than + /// sleeping, so a test never asserts on a clock. The message is what + /// the failure says, so it names what never happened. + /// + public static void Until( + Func condition, string whatWasWaitedFor) + { + var deadline = DateTime.UtcNow + Patience; + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return; + } + Thread.Sleep(100); + } + Assert.Fail( + $"Waited {Patience.TotalSeconds:0} seconds and " + + $"{whatWasWaitedFor} never happened."); + } + + #endregion +} diff --git a/Browser51Did/IsGdprAcceptanceTests.cs b/Browser51Did/IsGdprAcceptanceTests.cs new file mode 100644 index 0000000..35f6770 --- /dev/null +++ b/Browser51Did/IsGdprAcceptanceTests.cs @@ -0,0 +1,125 @@ +#nullable enable + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// Whether the regulation applies, and where the preference platform gets +/// that from. +/// +/// **These cannot pass until the change that puts IsGdpr in the cloud is +/// released**, being pipeline-dotnet pull request 413 and then cloud pull +/// request 372. The harness entitlement record already asks for the +/// property, so the moment the container carries it these run. Until then +/// the first test reports inconclusive with that reason rather than +/// failing, because a red test nobody can fix teaches a reader to ignore +/// red tests. +/// +/// +/// The dialog is shown and an identifier created either way, because the +/// question the platform asks is the Model Terms usage, which is a matter +/// of contract, and not a consent under the regulation. +/// +/// +[TestClass, TestCategory("Browser51Did")] +public class IsGdprAcceptanceTests : Browser51DidTestBase +{ + /// + /// With the property carrying a value, the platform's framework + /// surface reports it, and a false value does not stop the visitor + /// being asked. + /// + [TestMethod] + public void IsGdpr_ReadFromTheClientScript_SetsGdprApplies() + { + using var server = new PageServer(); + server.Put("/", Pages.Page( + "Is the regulation in force", + Pages.PlatformTag(new Pages.PlatformSettings()), + Pages.ClientScriptTag())); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForPlatform(); + visitor.WaitForClientRounds(1); + + var value = visitor.IsGdpr(); + if (value == "") + { + Assert.Inconclusive( + "The client script's object carries no isgdpr value, so " + + "there is nothing for the platform to read. The property " + + "reaches the cloud in pipeline-dotnet pull request 413 " + + "and then cloud pull request 372, and the harness " + + "entitlement record already asks for it, so this runs as " + + "soon as the container carries it."); + } + + var applies = visitor.GdprApplies(); + Assert.AreEqual( + string.Equals(value, "True", StringComparison.OrdinalIgnoreCase), + applies, + "the platform's framework surface must report what the client " + + $"script resolved. The script said '{value}' and the surface " + + $"said {applies}."); + + Assert.IsTrue( + visitor.CardVisible("preferences"), + "the visitor is asked whatever the answer is, because the " + + "question is the Model Terms usage and that is contractual " + + "rather than a consent under the regulation."); + + Assert.AreEqual( + 0, + visitor.ConsoleMatching("isgdpr").Count( + line => line.Contains( + "WARNING", StringComparison.OrdinalIgnoreCase)), + "with a value to read there is nothing to warn about. The " + + "console said: " + string.Join(" | ", visitor.Console())); + } + + /// + /// With no value to read, the platform says so once and carries on as + /// though the regulation applies, which is the safe way round. + /// + [TestMethod] + public void IsGdprAbsent_PlatformWarnsAndAssumesItApplies() + { + using var server = new PageServer(); + server.Put("/", Pages.Page( + "No isgdpr", + Pages.PlatformTag(new Pages.PlatformSettings()), + Pages.ClientScriptTag())); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForPlatform(); + visitor.WaitForClientRounds(1); + + if (visitor.IsGdpr() != "") + { + Assert.Inconclusive( + "The harness resource key does carry a value for isgdpr, " + + "so this is not the case under test. A key that does not " + + "ask for the property is needed, which the harness does " + + "not have a second of."); + } + + Assert.IsTrue( + visitor.GdprApplies(), + "with nothing to read, the platform assumes the regulation " + + "applies rather than assuming it does not."); + Assert.IsTrue( + visitor.ConsoleMatching("isgdpr").Count > 0, + "the platform says once that it could not read the property, " + + "because a publisher whose key does not carry it has no other " + + "way of finding out. The console said: " + + string.Join(" | ", visitor.Console())); + Assert.IsTrue( + visitor.CardVisible("preferences"), + "the dialog is still shown and an identifier still created."); + } +} diff --git a/Browser51Did/PageServer.cs b/Browser51Did/PageServer.cs new file mode 100644 index 0000000..c00c9c5 --- /dev/null +++ b/Browser51Did/PageServer.cs @@ -0,0 +1,193 @@ +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// The publisher's website, for the length of one test. It serves the page +/// fixtures and nothing else. +/// +/// It is deliberately not a proxy for the cloud. The pages load the client +/// script and the preference platform straight from the cloud's own origin, +/// which is what makes the cloud a third party to the page and the shared +/// cookie a third party cookie. A proxy would put both on one origin and +/// the first demonstration would pass for the wrong reason. What a test +/// needs to see of the traffic is recorded inside the browser instead, by +/// the recorder in , which reads the request bodies and +/// the responses the page actually sent and received. +/// +/// +/// It answers on a plain socket and pays no attention to the host name in +/// the request, rather than using the framework's own listener, because +/// that one routes by the name and registering any name but localhost +/// needs an administrator on Windows. A developer would then get a +/// different run from CI, which is the thing most likely to let a fault +/// through. Here every publisher site name reaches the same server, and +/// the browsers are told to resolve those names to this machine, so the +/// only thing separating one site from another is the name in the +/// request, which is exactly what a browser uses to decide what is third +/// party. +/// +/// +public sealed class PageServer : IDisposable +{ + private readonly TcpListener _listener; + private readonly CancellationTokenSource _stopping = new(); + private readonly ConcurrentDictionary _pages = new( + StringComparer.OrdinalIgnoreCase); + + /// + /// Method, host and path of every request served, newest last. A test + /// reads it while the server is live, so it is a queue that takes a + /// snapshot when enumerated rather than one that throws. + /// + public ConcurrentQueue RequestLog { get; } = new(); + + /// The port the sites are served on. + public int Port { get; } + + public PageServer() + { + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + _ = Task.Run(Accept); + } + + /// + /// Puts a page at a path. The same path may be replaced between + /// navigations, which is how a test changes what the second page view + /// carries. + /// + public void Put(string path, string html) + => _pages[Normalise(path)] = html; + + /// + /// The address of a page on one of the publisher sites. The cache + /// buster is on every navigation, because a page held in the browser's + /// cache would run the recorder from an earlier test. + /// + public string UrlFor(string site, string path) + => $"http://{site}:{Port}{Normalise(path)}" + + $"?v={DateTime.UtcNow.Ticks}"; + + /// Every request served since the last clear. + public IReadOnlyList Requests => new List(RequestLog); + + public void ClearRequests() => RequestLog.Clear(); + + private async Task Accept() + { + while (_stopping.IsCancellationRequested == false) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync( + _stopping.Token).ConfigureAwait(false); + } + catch (Exception) + { + // The listener was stopped, which is how a test ends. + return; + } + _ = Task.Run(() => Serve(client)); + } + } + + private async Task Serve(TcpClient client) + { + using (client) + { + try + { + using var stream = client.GetStream(); + using var reader = new StreamReader( + stream, Encoding.ASCII, false, 1024, leaveOpen: true); + var requestLine = await reader.ReadLineAsync() + .ConfigureAwait(false); + if (string.IsNullOrEmpty(requestLine)) + { + return; + } + var host = string.Empty; + string? header; + // Headers are read to the blank line so the browser's + // request is fully consumed, and the host is kept because + // the log is the one place a test can see which site a + // request was for. + while (string.IsNullOrEmpty( + header = await reader.ReadLineAsync().ConfigureAwait(false)) + == false) + { + if (header!.StartsWith( + "Host:", StringComparison.OrdinalIgnoreCase)) + { + host = header.Substring(5).Trim(); + } + } + var parts = requestLine.Split(' '); + var method = parts.Length > 0 ? parts[0] : "GET"; + var path = Normalise(parts.Length > 1 ? parts[1] : "/"); + RequestLog.Enqueue($"{method} {host}{path}"); + + var found = _pages.TryGetValue(path, out var html); + var body = Encoding.UTF8.GetBytes(found ? html! : "not here"); + var head = new StringBuilder() + .Append(found ? "HTTP/1.1 200 OK\r\n" + : "HTTP/1.1 404 Not Found\r\n") + .Append("Content-Type: text/html; charset=utf-8\r\n") + .Append("Content-Length: ") + .Append(body.Length) + .Append("\r\n") + // Nothing here may be reused between page views, or a + // test that navigates twice measures the first page + // view twice. + .Append("Cache-Control: no-store, no-cache, must-revalidate\r\n") + .Append("Connection: close\r\n\r\n") + .ToString(); + var headBytes = Encoding.ASCII.GetBytes(head); + await stream.WriteAsync(headBytes).ConfigureAwait(false); + await stream.WriteAsync(body).ConfigureAwait(false); + await stream.FlushAsync().ConfigureAwait(false); + } + catch (Exception) + { + // A browser that went away mid response is not a failure of + // the thing under test, and the assertions are on what the + // page recorded rather than on what this served. + } + } + } + + private static string Normalise(string path) + { + var trimmed = path.Split('?')[0].Split('#')[0]; + return trimmed.StartsWith("/", StringComparison.Ordinal) + ? trimmed + : "/" + trimmed; + } + + public void Dispose() + { + _stopping.Cancel(); + try + { + _listener.Stop(); + } + catch (Exception) + { + // Already stopped. + } + _stopping.Dispose(); + } +} diff --git a/Browser51Did/Pages.cs b/Browser51Did/Pages.cs new file mode 100644 index 0000000..5495cc6 --- /dev/null +++ b/Browser51Did/Pages.cs @@ -0,0 +1,324 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Text; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// The real pages a browser loads in these tests, being a publisher's page +/// with the preference platform and the client script on it in one +/// arrangement or another. +/// +/// Every page starts with the recorder, which is the only thing on any of +/// them that is not a publisher's own markup. It wraps the two ways a page +/// makes a request and the console, so a test can read what the page +/// actually sent, what came back and what was logged, without a proxy in +/// front of the cloud. A proxy would put the cloud on the page's own +/// origin, and the whole first demonstration is about the cloud being a +/// third party to the page. +/// +/// +/// Nothing on these pages is publisher code in the sense the design means +/// it. The recorder is the test's instrument, and where a page carries a +/// handler, the test that uses that page says why. +/// +/// +public static class Pages +{ + /// + /// The object name a page uses when it does not ask for another one, + /// which is what the client script and the preference platform both + /// fall back to. + /// + public const string DefaultObjectName = "fod"; + + /// + /// A publisher's static vendor consent string, as the platform's own + /// demo page carries one. The platform only rewrites the purpose + /// consent bits and the dates in it. + /// + public const string PublisherVendorString = + "CPYBSvoPYBSvoO3AAAENAwCAAAAAAAAAAAAAAAAAAAAA"; + + /// + /// The recorder. It is first on every page, before anything is loaded + /// from the cloud, so nothing the page does afterwards escapes it. + /// + public const string Recorder = @" +"; + + /// + /// A handler registered on the client script's object as soon as it + /// exists, so a test can assert that a change of answer reached page + /// code. It waits for the object rather than assuming the script has + /// finished, because the script tag is asynchronous. + /// + /// This is the one piece of publisher code any of these pages carries, + /// and only the two tests about a change of answer use it. + /// + /// + public static string ChangeWatcher(string objectName) => $@" +"; + + /// + /// The client script's own tag, exactly as a publisher writes it. + /// + public static string ClientScriptTag(string? objectName = null) + => $""; + + /// + /// Settings on the preference platform's tag. Every one of them is an + /// attribute the publisher writes, and the defaults here are the ones + /// the platform's own demo page uses. + /// + public sealed class PlatformSettings + { + /// The network a choice is shared across. Null turns + /// sharing off, because the visitor cannot be asked to share with + /// a group nobody has named. + public string? NetworkName { get; set; } = Harness.NetworkName; + + /// Whether the standard answer is offered as well as the + /// personalized one. + public bool ShowStandard { get; set; } = true; + + /// The name the platform looks for the client script's + /// object under. Null leaves the attribute off, which is the + /// documented way of saying the default. + public string? ObjectName { get; set; } + + /// How long the platform waits for the third party cookie + /// answer before carrying on without one. + public int TimeoutMs { get; set; } = 4500; + } + + /// + /// The preference platform's tag. The action and the alternative are + /// page hooks rather than a second script, so a test can see them fire + /// and so that nothing here loads a second copy of the client script, + /// which two of the tests assert did not happen. + /// + public static string PlatformTag(PlatformSettings settings) + { + var attributes = new List + { + $"data-resource-key=\"{Harness.Resource}\"", + "data-action-url=\"javascript:window.__51dTest.actions" + + ".push('{preference}')\"", + $"data-tcf-vendor=\"{PublisherVendorString}\"", + "data-brand-name=\"Fifty One Times\"", + "data-brand-terms-url=\"https://example.com/privacy\"", + "data-alt-name=\"Subscribe\"", + "data-alt-url=\"javascript:window.__51dTest.altFired = true\"", + $"data-show-standard=\"{(settings.ShowStandard ? "true" : "false")}\"", + $"data-timeout=\"{settings.TimeoutMs}\"", + }; + if (settings.NetworkName is null) + { + attributes.Add("data-use-third-party-cookies=\"false\""); + } + else + { + attributes.Add($"data-network-name=\"{settings.NetworkName}\""); + } + if (settings.ObjectName is not null) + { + attributes.Add($"data-object-name=\"{settings.ObjectName}\""); + } + return $""; + } + + /// + /// A consent platform, being the stub the framework's own + /// specification tells every publisher to put in the page, and a fake + /// platform behind it that answers only what the specification + /// requires, which is ping, addEventListener and a callback carrying + /// tcString and eventStatus. + /// + /// It does not deliver anything until the test asks it to with + /// window.__51dCmp.deliver(), so the test decides the ordering + /// rather than a timer. + /// + /// + public static string ConsentPlatform(string tcString) => $@" +"; + + /// Wraps the parts into a page. + public static string Page(string title, params string[] parts) + { + var body = new StringBuilder(); + body.Append("\n\n\n") + .Append("\n") + .Append("").Append(title).Append("\n") + .Append(Recorder) + .Append("\n\n\n

") + .Append(title) + .Append("

\n"); + foreach (var part in parts) + { + body.Append(part).Append('\n'); + } + return body.Append("\n\n").ToString(); + } +} diff --git a/Browser51Did/PlatformAcceptanceTests.cs b/Browser51Did/PlatformAcceptanceTests.cs new file mode 100644 index 0000000..58fe854 --- /dev/null +++ b/Browser51Did/PlatformAcceptanceTests.cs @@ -0,0 +1,556 @@ +#nullable enable + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using FiftyOne.Did.Model; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// The preference platform and the client script on one page, which is the +/// arrangement the design is built around, proved in a real browser. +/// +/// These are demonstrations 1, 3 and 4 of the acceptance list, with the +/// change of answer and the alternative answer that came from the later +/// decisions. Every assertion is on an ordering of requests and page +/// states, never on a clock. +/// +/// +[TestClass, TestCategory("Browser51Did")] +public class PlatformAcceptanceTests : Browser51DidTestBase +{ + /// + /// Demonstration 3, and the one that carries the most. A first visit, + /// the visitor answers the first card, the answer reaches the client + /// script, the full sequence runs with the usage known and an + /// identifier comes back with the signal source recorded as direct. + /// The second card is up before any of that finished, because nothing + /// waits between the two cards. + /// + [TestMethod] + public void CommonPath_PlatformThenScript_Chrome() + => CommonPath(Chrome, platformFirst: true); + + /// + /// The same, in the other browser. Demonstration 9 asks for both, + /// because anything the shared choice travels on behaves differently + /// between them. + /// + [TestMethod] + public void CommonPath_PlatformThenScript_Firefox() + => CommonPath(Firefox, platformFirst: true); + + /// + /// The same page with the two tags the other way round. The platform's + /// bundle loads on its own timetable, so the announcement has to reach + /// the client script whichever tag the publisher wrote first, and a + /// design that only worked one way round would pass the test above and + /// fail on half the customers' pages. + /// + [TestMethod] + public void CommonPath_ScriptThenPlatform_Chrome() + => CommonPath(Chrome, platformFirst: false); + + private void CommonPath(string browser, bool platformFirst) + { + using var server = new PageServer(); + RequireMarketingIdentifiers(); + var settings = new Pages.PlatformSettings(); + var platform = Pages.PlatformTag(settings); + var script = Pages.ClientScriptTag(); + server.Put("/", Pages.Page( + "Common path", + platformFirst ? platform : script, + platformFirst ? script : platform)); + + using var visitor = NewVisitor(browser, server); + visitor.Go(Harness.SiteA, "/"); + + // The first round. Nobody has been asked yet, so nothing may be + // created, which is the rule the whole programme exists for. + visitor.WaitForClientRounds(1); + Assert.IsTrue( + visitor.HasTheNewClientScript(), + "the page loaded a client script with no user prompt block in " + + "it, so this test would be proving the old behaviour. The " + + "guard passed, so the container served the new script and " + + "something else on the page loaded an old one."); + var first = visitor.ClientRequests()[0]; + Assert.IsNull( + first.Form("id.usage"), + "the first request went before anyone was asked, so it must " + + $"carry no answer. It carried: {first.Body}"); + Assert.AreEqual( + "", + visitor.Identifier(), + "no answer means no identifier, however much else the cloud " + + "resolved."); + + // The visitor answers. + visitor.WaitForPlatform(); + visitor.WaitForCard("preferences"); + var roundsBefore = visitor.ClientRequests().Count; + // How many rounds had already finished, which is what the share + // card must appear before any more of. The page may well have + // finished more than one by now, because the snippets it was asked + // to run produce a round of their own, so this is read rather than + // assumed to be one. + var doneBefore = visitor.ClientRequests().Count(r => r.Done); + visitor.Press("standard"); + + // The second card is up before the refresh has finished. Both + // readings come from one call, so there is no gap between them in + // which the answer could change. + (bool CardVisible, int RoundsDone) atShare = (false, -1); + Harness.Until( + () => + { + var seen = visitor.Observe("share"); + if (seen.CardVisible && atShare.RoundsDone < 0) + { + atShare = seen; + } + return atShare.RoundsDone >= 0; + }, + $"the share card appeared in {browser}. The console said: " + + $"{string.Join(" | ", visitor.Console())}"); + Assert.AreEqual( + doneBefore, + atShare.RoundsDone, + "the share card must follow the first card at once, with " + + "nothing waiting on the refresh. When it appeared the client " + + $"script had finished {atShare.RoundsDone} rounds rather than " + + $"the {doneBefore} it had finished at the click, so something " + + "waited."); + + // Exactly one further request, carrying the answer once and every + // snippet result the page had worked out. + visitor.WaitForClientRounds(roundsBefore + 1); + var requests = visitor.ClientRequests(); + Assert.AreEqual( + roundsBefore + 1, + requests.Count, + "answering the first card must produce exactly one further " + + "request. The scripts on the page were: " + + string.Join(", ", visitor.ScriptSources()) + + ". The console said: " + + string.Join(" | ", visitor.Console()) + + ". The requests were: " + + string.Join(" || ", requests.Select(r => r.ToString()))); + var answered = requests[requests.Count - 1]; + Assert.AreEqual( + "standard", + answered.Form("id.usage"), + $"the answer must reach the cloud. The body was: {answered.Body}"); + Assert.AreEqual( + 1, + answered.FormCount("id.usage"), + "the answer must be sent once. The server takes the first value " + + "of a repeated key and warns, so sending it twice would " + + $"create nothing. The body was: {answered.Body}"); + foreach (var snippet in visitor.SnippetValueNames()) + { + Assert.IsNotNull( + answered.Form(snippet), + $"the request that creates the identifier must carry every " + + $"snippet result. '{snippet}' was stored by the page and " + + $"not sent. The body was: {answered.Body}"); + } + + // The identifier, and the flag that says the answer was stated + // rather than decoded from a framework string. + var identifier = visitor.Identifier(); + AssertDirect( + identifier, + "an answer given on the platform is stated directly"); + Assert.AreEqual( + Usage.Standard, + UsageOf(identifier, "the answer was standard"), + "the visitor pressed standard."); + Assert.IsTrue( + answered.Response.Contains("fodid", StringComparison.Ordinal), + "the response to the answered request must carry the identifier " + + $"section. It was: {Truncate(answered.Response)}"); + + // Nothing loaded a second copy of the client script. + Assert.AreEqual( + 0, + visitor.ConsoleMatching(Harness.SecondInstanceWarning).Count, + "a second instance of the client script was loaded onto the " + + "page. The console said: " + + string.Join(" | ", visitor.Console())); + + // And the shared write, which is the second card's whole purpose. + RequireSharing(); + visitor.Press("share-accept"); + Harness.Until( + () => visitor.SharedStoreRequests() + .Any(r => r.Method == "POST" && r.Done), + $"the shared choice was written in {browser}. The console said: " + + $"{string.Join(" | ", visitor.Console())}"); + var write = visitor.SharedStoreRequests() + .Last(r => r.Method == "POST"); + Assert.AreEqual( + 200, + write.Status, + "the cloud must accept the shared write. It answered " + + $"{write.Status}: {Truncate(write.Response)}"); + } + + /// + /// Demonstration 1. A choice made on one site is read on another, the + /// visitor is not asked again, and the client script still creates an + /// identifier from the answer with the signal source recorded as + /// direct. + /// + [TestMethod] + public void SharedChoice_SecondSite_NoDialogAndDirectFlag_Chrome() + => SharedChoice(Chrome); + + /// + /// The same across browsers, because the shared choice travels on a + /// third party cookie and that is the thing the two browsers treat + /// differently. + /// + [TestMethod] + public void SharedChoice_SecondSite_NoDialogAndDirectFlag_Firefox() + => SharedChoice(Firefox); + + private void SharedChoice(string browser) + { + using var server = new PageServer(); + RequireMarketingIdentifiers(); + RequireSharing(); + var page = Pages.Page( + "Shared choice", + Pages.PlatformTag(new Pages.PlatformSettings()), + Pages.ClientScriptTag()); + server.Put("/", page); + + using var visitor = NewVisitor(browser, server); + + // Site A, where the visitor answers and agrees to share. + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForCard("preferences"); + visitor.Press("standard"); + visitor.WaitForCard("share"); + visitor.Press("share-accept"); + Harness.Until( + () => visitor.SharedStoreRequests() + .Any(r => r.Method == "POST" && r.Done && r.Status == 200), + $"the choice was shared from {Harness.SiteA} in {browser}. The " + + $"console said: {string.Join(" | ", visitor.Console())}"); + + // Site B, a first visit, in the same browser. + visitor.Go(Harness.SiteB, "/"); + visitor.WaitForPlatform(); + visitor.WaitForClientRounds(1); + + var read = visitor.SharedStoreRequests() + .FirstOrDefault(r => r.Method == "GET" && r.Done); + Assert.IsNotNull( + read, + "the platform must ask the shared store what this visitor " + + "already chose. It made no such call. The requests were: " + + string.Join(" || ", + visitor.Requests().Select(r => r.ToString()))); + Assert.IsTrue( + read!.Response.Contains("standard", StringComparison.Ordinal), + "the shared store must answer with the choice made on the other " + + $"site. It answered: {Truncate(read.Response)}"); + + Assert.IsTrue( + visitor.BubbleOnly(), + "a visitor who has already answered is not asked again, so only " + + $"the floating button shows. The dialog is: {visitor.PlatformState()}" + + ". The console said: " + + string.Join(" | ", visitor.Console())); + + Harness.Until( + () => visitor.ClientRequests() + .Any(r => r.Done && r.Form("id.usage") == "standard"), + "the answer read from the shared store reached the client " + + $"script in {browser}. The console said: " + + $"{string.Join(" | ", visitor.Console())}"); + var answered = visitor.ClientRequests() + .Last(r => r.Form("id.usage") == "standard"); + Assert.AreEqual( + 1, + answered.FormCount("id.usage"), + "the answer must be sent once, or the server drops the repeat " + + $"and creates nothing. The body was: {answered.Body}"); + + var identifier = visitor.Identifier(); + AssertDirect( + identifier, + "a choice made on the platform stays a stated usage on the " + + "second site"); + Assert.AreEqual( + Usage.Standard, + UsageOf(identifier, "the choice was standard"), + "the choice carried across is the one that was made."); + + Assert.AreEqual( + "standard", + visitor.PlatformPreference(), + "the platform's own getter must answer with the choice it is " + + "acting on, on the second site as much as on the first."); + + var stored = visitor.LocalStorageKeys(); + Assert.AreEqual( + 0, + stored.Count, + "nothing new is written to browser storage by the platform, and " + + "the second site's answer lives in the shared store rather " + + "than being copied here. It wrote: " + + string.Join(", ", stored)); + } + + /// + /// A change of answer on the same page. The sequence goes up, a + /// different identifier comes back, and page code that registered a + /// change handler is told. + /// + [TestMethod] + public void ChangeOfAnswer_SamePage_NewIdentifierAndOnChange() + { + using var server = new PageServer(); + RequireMarketingIdentifiers(); + server.Put("/", Pages.Page( + "Change of answer", + Pages.PlatformTag(new Pages.PlatformSettings()), + Pages.ClientScriptTag(), + Pages.ChangeWatcher(Pages.DefaultObjectName))); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForCard("preferences"); + visitor.Press("standard"); + Harness.Until( + () => visitor.Identifier() != "", + "the first answer produced an identifier. The console said: " + + string.Join(" | ", visitor.Console())); + var firstIdentifier = visitor.Identifier(); + var roundsBefore = visitor.ClientRequests().Count; + var sequenceBefore = Sequence(visitor.ClientRequests().Last()); + + // The visitor changes their mind, through the platform. + visitor.OpenPlatform(); + visitor.WaitForCard("preferences"); + visitor.Press("personalized"); + + Harness.Until( + () => visitor.Identifier() != "" + && visitor.Identifier() != firstIdentifier, + "a different identifier came back for the changed answer. The " + + "console said: " + string.Join(" | ", visitor.Console())); + var requests = visitor.ClientRequests(); + Assert.AreEqual( + roundsBefore + 1, + requests.Count, + "a change of answer makes exactly one further request. The " + + "scripts on the page were: " + + string.Join(", ", visitor.ScriptSources()) + + ". The console said: " + + string.Join(" | ", visitor.Console()) + + ". The requests were: " + + string.Join(" || ", requests.Select(r => r.ToString()))); + var changed = requests[requests.Count - 1]; + Assert.AreEqual( + "personalized", + changed.Form("id.usage"), + $"the new answer must be the one sent. The body was: {changed.Body}"); + Assert.AreEqual( + sequenceBefore + 1, + Sequence(changed), + "the sequence goes up by one for the round that carried the " + + $"changed answer. The body was: {changed.Body}"); + + var second = visitor.Identifier(); + AssertDirect(second, "the changed answer was stated directly"); + Assert.AreEqual( + Usage.Personalized, + UsageOf(second, "the changed answer was personalized"), + "the identifier must carry the answer that was actually given."); + Assert.AreNotEqual( + firstIdentifier, + second, + "a change of answer produces a new identifier."); + + var told = visitor.ChangeIdentifiers(); + Assert.IsTrue( + told.Contains(second), + "page code that registered a change handler before the change " + + "must be told about the new identifier. It was told: " + + string.Join(", ", told)); + } + + /// + /// A change of answer carried across two pages in one tab. The second + /// page must never serve the visitor the answer they moved away from, + /// which is the whole point of the record the client script keeps. + /// + /// What is asserted is the outcome and not the number of requests. The + /// stored record is the inputs of the last request, and a page view + /// whose inputs match it is meant to reuse the answer rather than ask + /// again, which is the rule about reuse working rather than failing. + /// Changing the answer on the first page makes that page ask again and + /// re-record, so the second page's inputs match the new record and it + /// may legitimately reuse it. Measured against a running service on + /// 15 September 2026, that is what happens. Asserting a fresh request + /// here would be asserting that the cache does not work. + /// + /// + [TestMethod] + public void ChangeOfAnswer_AcrossPages_TheSecondPageCarriesTheNewAnswer() + { + using var server = new PageServer(); + RequireMarketingIdentifiers(); + var page = Pages.Page( + "Two pages", + Pages.PlatformTag(new Pages.PlatformSettings()), + Pages.ClientScriptTag(), + Pages.ChangeWatcher(Pages.DefaultObjectName)); + server.Put("/one", page); + server.Put("/two", page); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/one"); + visitor.WaitForCard("preferences"); + visitor.Press("standard"); + Harness.Until( + () => visitor.Identifier() != "", + "the first page produced an identifier. The console said: " + + string.Join(" | ", visitor.Console())); + var underStandard = visitor.Identifier(); + Assert.AreEqual( + Usage.Standard, + UsageOf(underStandard, "the first answer"), + "the visitor pressed standard first."); + + // The answer is changed before the visitor leaves the first page. + visitor.OpenPlatform(); + visitor.WaitForCard("preferences"); + visitor.Press("personalized"); + Harness.Until( + () => visitor.Identifier() != underStandard + && visitor.Identifier() != "", + "the changed answer took effect on the first page. The console " + + "said: " + string.Join(" | ", visitor.Console())); + Assert.AreEqual( + Usage.Personalized, + UsageOf(visitor.Identifier(), "the changed answer"), + "the change was to personalized."); + + // The second page, in the same tab. + visitor.Go(Harness.SiteA, "/two"); + Harness.Until( + () => visitor.Identifier() != "", + "the second page settled on an identifier. The console said: " + + string.Join(" | ", visitor.Console())); + var onPageTwo = visitor.Identifier(); + + Assert.AreEqual( + Usage.Personalized, + UsageOf(onPageTwo, "the second page's identifier"), + "the answer in force when the second page loaded is the one it " + + "must carry, whether it asked again or reused the record " + + "written after the change."); + Assert.AreNotEqual( + underStandard, + onPageTwo, + "the second page must never be serving the identifier made " + + "under the answer the visitor moved away from."); + + var told = visitor.ChangeIdentifiers(); + Assert.IsTrue( + told.Contains(onPageTwo), + "the change handler on the new instance must be told about the " + + "identifier it settled on, including where that came from the " + + "record rather than from a fresh request. It was told: " + + string.Join(", ", told)); + } + + /// + /// The alternative answer. It is an answer under the Model Terms like + /// any other, so it creates an identifier with the direct flag, and + /// there is nothing to share, so no second card is offered. + /// + [TestMethod] + public void AlternativeAnswer_CreatesNonMarketingAndFiresTheAction() + { + using var server = new PageServer(); + server.Put("/", Pages.Page( + "Alternative", + Pages.PlatformTag(new Pages.PlatformSettings()), + Pages.ClientScriptTag())); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForCard("preferences"); + visitor.Press("alternative"); + + Harness.Until( + visitor.AlternativeFired, + "the action the publisher configured for the alternative " + + "button fired. The console said: " + + string.Join(" | ", visitor.Console())); + Harness.Until( + () => visitor.CardVisible("preferences") == false, + "the dialog closed after the alternative was pressed. The " + + "console said: " + string.Join(" | ", visitor.Console())); + Assert.IsFalse( + visitor.CardVisible("share"), + "there is nothing to share, so the second card must never be " + + "offered after the alternative."); + + Harness.Until( + () => visitor.ClientRequests() + .Any(r => r.Done && r.Form("id.usage") == "non-marketing"), + "the alternative answer reached the cloud. The console said: " + + string.Join(" | ", visitor.Console())); + var answered = visitor.ClientRequests() + .Last(r => r.Form("id.usage") == "non-marketing"); + Assert.AreEqual( + 1, + answered.FormCount("id.usage"), + $"the answer must be sent once. The body was: {answered.Body}"); + + Harness.Until( + () => visitor.Identifier() != "", + "an identifier came back for the alternative answer. The " + + "console said: " + string.Join(" | ", visitor.Console())); + var identifier = visitor.Identifier(); + AssertDirect( + identifier, + "the alternative is an answer the visitor gave directly"); + Assert.AreEqual( + Usage.NonMarketing, + UsageOf(identifier, "the alternative answer"), + "the alternative stores non-marketing."); + + // The framework surface answers (null, false) after the + // alternative, which is the framework's view of a usage granting + // no purposes. It is observed here so that the difference between + // it and what the request carried is on the record. + Assert.AreEqual( + "non-marketing", + visitor.PlatformPreference(), + "the platform is holding an answer even though its framework " + + "surface reports none, and the answer is what creates the " + + "identifier."); + } + + private static int Sequence(RecordedRequest request) + => int.TryParse( + request.Form("sequence"), + System.Globalization.NumberStyles.Integer, + System.Globalization.CultureInfo.InvariantCulture, + out var value) + ? value + : 0; + + private static string Truncate(string value) + => value.Length <= 300 ? value : value.Substring(0, 300) + "..."; +} diff --git a/Browser51Did/PlatformAddsClientScriptTests.cs b/Browser51Did/PlatformAddsClientScriptTests.cs new file mode 100644 index 0000000..51fb5d8 --- /dev/null +++ b/Browser51Did/PlatformAddsClientScriptTests.cs @@ -0,0 +1,234 @@ +#nullable enable + +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using FiftyOne.Did.Model; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// A page carrying the preference platform's tag and no client script tag +/// at all. +/// +/// James Rosewell, 15 September 2026. The platform has one route to the +/// third party cookie result and to whether the regulation applies, which +/// is the client script's object. Where that object is not on the page the +/// platform adds the script itself, using the cloud that served it and the +/// resource key it already holds, and says in the console that it did. A +/// second route would be a second answer to the same question, and a +/// publisher who forgot the tag would get a dialog behaving differently +/// from the documented one with nothing to tell them why. +/// +/// +[TestClass, TestCategory("Browser51Did")] +public class PlatformAddsClientScriptTests : Browser51DidTestBase +{ + /// + /// The whole of it in one page view, from the script arriving to the + /// identifier coming back. + /// + [TestMethod] + public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() + { + using var server = new PageServer(); + RequireMarketingIdentifiers(); + server.Put("/", Pages.Page( + "Platform with no client script", + Pages.PlatformTag(new Pages.PlatformSettings()))); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForPlatform(); + + // The script the platform added, named by where it came from and + // which publisher it is for. + Harness.Until( + () => AddedClientScript(visitor) != null, + "the platform added the client script. The console said: " + + string.Join(" | ", visitor.Console())); + var added = AddedClientScript(visitor)!; + Assert.IsTrue( + added.StartsWith(Harness.BaseUrl!, StringComparison.OrdinalIgnoreCase), + "the script must come from the cloud that served the platform, " + + "because that is the only cloud the platform knows about. It " + + $"came from {added}."); + Assert.IsTrue( + added.Contains(Harness.Resource!, StringComparison.Ordinal), + "the script must be asked for with the resource key the " + + "platform already holds, or the cloud has no idea which " + + $"publisher is asking. The URL was {added}."); + + // And it said so, which is the point of the convenience. + Assert.IsTrue( + visitor.ConsoleMatching(Harness.NoClientScriptMessage).Count > 0, + "the platform must say in the console that there was no client " + + "script object on the page, because a publisher who left the " + + "tag out has no other way of finding out. The console said: " + + string.Join(" | ", visitor.Console())); + Assert.IsTrue( + visitor.ConsoleMatching(Harness.AddingClientScriptMessage).Count > 0, + "and that it is adding one, so the publisher knows where the " + + "extra request came from. The console said: " + + string.Join(" | ", visitor.Console())); + + // The object then exists, under the name in force. + visitor.WaitForClientObject(); + Assert.IsTrue( + visitor.HasTheNewClientScript(), + "the script the platform added must be the new one, or the " + + "platform has quietly given itself the old behaviour."); + + // The third party cookie result reached the platform through it, + // which is the reason the script is added at all. + Harness.Until( + () => visitor.ThirdPartyCookies() != "", + "the client script resolved the third party cookie result. The " + + "console said: " + string.Join(" | ", visitor.Console())); + + // And the answer still creates, exactly as it would have on a page + // that carried the tag. + visitor.WaitForCard("preferences"); + visitor.Press("standard"); + Harness.Until( + () => visitor.ClientRequests() + .Any(r => r.Done && r.Form("id.usage") == "standard"), + "the answer reached the cloud through the script the platform " + + "added. The console said: " + + string.Join(" | ", visitor.Console())); + var answered = visitor.ClientRequests() + .Last(r => r.Form("id.usage") == "standard"); + Assert.AreEqual( + 1, + answered.FormCount("id.usage"), + $"the answer must be sent once. The body was: {answered.Body}"); + foreach (var snippet in visitor.SnippetValueNames()) + { + Assert.IsNotNull( + answered.Form(snippet), + "the request that creates the identifier must carry every " + + $"snippet result. '{snippet}' was stored and not sent. " + + $"The body was: {answered.Body}"); + } + + Harness.Until( + () => visitor.Identifier() != "", + "an identifier came back. The console said: " + + string.Join(" | ", visitor.Console())); + AssertDirect( + visitor.Identifier(), + "an answer given on the platform is stated directly"); + Assert.AreEqual( + Usage.Standard, + UsageOf(visitor.Identifier(), "the answer"), + "the visitor pressed standard."); + + // One script, not two. The platform adds one only where there is + // none, so nothing here may trip the warning about a second copy. + Assert.AreEqual( + 0, + visitor.ConsoleMatching(Harness.SecondInstanceWarning).Count, + "the platform added a second copy of the client script, or " + + "added one to a page that already had it. The console said: " + + string.Join(" | ", visitor.Console())); + Assert.AreEqual( + 1, + ClientScriptsOnThePage(visitor), + "there must be exactly one client script on the page. Its " + + "sources were: " + string.Join(", ", visitor.ScriptSources())); + } + + /// + /// The publisher may name the object something other than the default, + /// and the platform then asks for the script under that name, finds it + /// under that name, and says which name it used. + /// + [TestMethod] + public void ObjectNameAttribute_NamesTheObjectEverywhere() + { + const string objectName = "fiftyOneData"; + using var server = new PageServer(); + server.Put("/", Pages.Page( + "Platform with a named object", + Pages.PlatformTag(new Pages.PlatformSettings + { + ObjectName = objectName, + }))); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForPlatform(); + + Harness.Until( + () => AddedClientScript(visitor) != null, + "the platform added the client script. The console said: " + + string.Join(" | ", visitor.Console())); + var added = AddedClientScript(visitor)!; + Assert.IsTrue( + added.Contains(objectName, StringComparison.Ordinal), + "the name the publisher asked for must be on the URL, or the " + + "cloud renders the script under the default name and the " + + $"platform then looks for the wrong object. The URL was {added}."); + + visitor.WaitForClientObject(objectName); + Assert.IsTrue( + visitor.HasTheNewClientScript(objectName), + $"the object under '{objectName}' must be the client script's."); + Assert.IsFalse( + visitor.HasClientObject(Pages.DefaultObjectName), + "nothing may be put under the default name when the publisher " + + "asked for another one, because two objects would be two " + + "instances and the page would warn about the second."); + + var named = visitor.ConsoleMatching(objectName); + Assert.IsTrue( + named.Count > 0, + "every message about the object names whichever name is in " + + "force, so a publisher reading the console can tell which " + + "object is being talked about. The console said: " + + string.Join(" | ", visitor.Console())); + } + + /// + /// The attribute left off uses the default name, which is what the + /// documentation says and what an existing page relies on. + /// + [TestMethod] + public void ObjectNameAttributeAbsent_UsesTheDefaultName() + { + using var server = new PageServer(); + server.Put("/", Pages.Page( + "Platform with no object name", + Pages.PlatformTag(new Pages.PlatformSettings()))); + + using var visitor = NewVisitor(Chrome, server); + visitor.Go(Harness.SiteA, "/"); + visitor.WaitForPlatform(); + visitor.WaitForClientObject(Pages.DefaultObjectName); + Assert.IsTrue( + visitor.HasTheNewClientScript(Pages.DefaultObjectName), + "with no name asked for, the object is the default one."); + } + + /// + /// The client script the platform added, or null where it has not + /// added one yet. Anything the cloud serves as a resource key script + /// counts, and the page carried none of its own, so whatever is there + /// was added. + /// + private static string? AddedClientScript(Visitor visitor) + => visitor.ScriptSources() + .FirstOrDefault(src => + src.Contains("/api/v4/", StringComparison.OrdinalIgnoreCase) + && src.Contains(".js", StringComparison.OrdinalIgnoreCase) + && src.Contains("/pmp", StringComparison.OrdinalIgnoreCase) + == false); + + private static int ClientScriptsOnThePage(Visitor visitor) + => visitor.ScriptSources() + .Count(src => + src.Contains("/api/v4/", StringComparison.OrdinalIgnoreCase) + && src.Contains(".js", StringComparison.OrdinalIgnoreCase) + && src.Contains("/pmp", StringComparison.OrdinalIgnoreCase) + == false); +} diff --git a/Browser51Did/TcString.cs b/Browser51Did/TcString.cs new file mode 100644 index 0000000..11ea99f --- /dev/null +++ b/Browser51Did/TcString.cs @@ -0,0 +1,76 @@ +#nullable enable + +using System; +using System.Collections.Generic; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// The smallest framework string that says which purposes a visitor +/// granted. Only the purpose consent bits and the legitimate interest bits +/// are set, because those are the only ones the server reads when it works +/// out a usage from a string. +/// +/// This is the same construction as TcStringBuilder in +/// Did/Tests/FiftyOne.Did.OnPremise.Tests. It is repeated here +/// rather than referenced because this project is deliberately outside the +/// solution and takes no project references, and because a browser test +/// that quietly changed when a unit test helper changed would be worse +/// than a small repetition. +/// +/// +public static class TcString +{ + private const int ConsentStart = 152; + private const int LegitimateInterestStart = 176; + private const int PurposeCount = 12; + + /// + /// Enough bytes for both purpose runs, which is the whole of the core + /// segment the reader looks at. + /// + private const int Bytes = 24; + + /// + /// The purposes the standard usage under the Model Terms for + /// Marketing needs. + /// + public static readonly int[] StandardPurposes = { 1, 2, 7, 8, 11 }; + + /// + /// Every purpose, which is what the personalized usage needs. + /// + public static readonly int[] AllPurposes = + { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }; + + /// A string granting exactly the purposes given. + public static string ForPurposes(IEnumerable consented) + { + var bytes = new byte[Bytes]; + foreach (var purpose in consented) + { + if (purpose < 1 || purpose > PurposeCount) + { + throw new ArgumentOutOfRangeException(nameof(consented)); + } + var bit = ConsentStart + purpose - 1; + bytes[bit / 8] |= (byte)(1 << (7 - (bit % 8))); + } + // Nothing is granted on legitimate interest, so the run starting + // here stays clear. It is named so the layout is readable rather + // than being a number nobody can check. + _ = LegitimateInterestStart; + return Convert.ToBase64String(bytes) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + } + + /// A string granting everything, which decodes to + /// personalized. + public static string Personalized() => ForPurposes(AllPurposes); + + /// A string granting the standard set, which decodes to + /// standard. + public static string Standard() => ForPurposes(StandardPurposes); +} diff --git a/Browser51Did/Visitor.cs b/Browser51Did/Visitor.cs new file mode 100644 index 0000000..4f8c1ee --- /dev/null +++ b/Browser51Did/Visitor.cs @@ -0,0 +1,588 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// One request a page made, as the page itself saw it. +public sealed class RecordedRequest +{ + [JsonPropertyName("kind")] public string Kind { get; set; } = ""; + [JsonPropertyName("method")] public string Method { get; set; } = ""; + [JsonPropertyName("url")] public string Url { get; set; } = ""; + [JsonPropertyName("body")] public string Body { get; set; } = ""; + [JsonPropertyName("status")] public int Status { get; set; } + [JsonPropertyName("response")] public string Response { get; set; } = ""; + [JsonPropertyName("done")] public bool Done { get; set; } + [JsonPropertyName("error")] public string? Error { get; set; } + + /// + /// The value of a form key in the request body, or null where the key + /// is absent. The body is form encoded with spaces as plus signs, + /// which is what the client script sends. + /// + public string? Form(string key) + { + foreach (var pair in Body.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var split = pair.IndexOf('='); + var name = split < 0 ? pair : pair.Substring(0, split); + if (Uri.UnescapeDataString(name.Replace('+', ' ')) != key) + { + continue; + } + var value = split < 0 ? "" : pair.Substring(split + 1); + return Uri.UnescapeDataString(value.Replace('+', ' ')); + } + return null; + } + + /// How many times a form key appears in the body. + public int FormCount(string key) + => Body.Split('&', StringSplitOptions.RemoveEmptyEntries) + .Count(pair => + { + var split = pair.IndexOf('='); + var name = split < 0 ? pair : pair.Substring(0, split); + return Uri.UnescapeDataString(name.Replace('+', ' ')) == key; + }); + + public override string ToString() + => $"{Kind} {Method} {Url} status={Status} done={Done} " + + $"body={Body}"; +} + +/// +/// One visitor, being a browser and the publisher's website it is looking +/// at. Everything a test asserts on is read back through here, so an +/// assertion reads as a sentence about the page rather than as a lump of +/// JavaScript. +/// +public sealed class Visitor : IDisposable +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNameCaseInsensitive = true, + }; + + /// + /// Finding the preference platform's dialog. It renders into a shadow + /// root attached to a plain div with no name of its own, so the root + /// is found by what is inside it. + /// + private const string Prelude = @" + function pmpRoot() { + var all = document.querySelectorAll('*'); + for (var i = 0; i < all.length; i++) { + var r = all[i].shadowRoot; + if (r && r.querySelector('.pmp')) { return r; } + } + return null; + } + // Whether a visitor can actually see something. offsetParent alone + // is not enough, because a browser reports null for it on anything + // positioned fixed, which the dialog is, and the answer would then + // be no for a card that is plainly on the screen. + function visible(el) { + if (!el) { return false; } + var style = window.getComputedStyle(el); + if (style.display === 'none' + || style.visibility === 'hidden' + || Number(style.opacity) === 0) { + return false; + } + var box = el.getBoundingClientRect(); + return box.width > 0 && box.height > 0; + } + "; + + public IWebDriver Driver { get; } + + public PageServer Server { get; } + + /// The browser's name, for a failure message. + public string BrowserName { get; } + + public Visitor(IWebDriver driver, PageServer server, string browserName) + { + Driver = driver; + Server = server; + BrowserName = browserName; + } + + /// Loads a page and waits for the document to settle. + public void Go(string site, string path) + { + Driver.Navigate().GoToUrl(Server.UrlFor(site, path)); + Harness.Until( + () => Script("return document.readyState;") == "complete", + $"the page at {site}{path} finished loading"); + } + + public T? Script(string body, params object[] arguments) + { + var result = ((IJavaScriptExecutor)Driver) + .ExecuteScript(Prelude + body, arguments); + if (result is null) + { + return default; + } + if (result is T typed) + { + return typed; + } + return (T)Convert.ChangeType( + result, typeof(T), CultureInfo.InvariantCulture); + } + + private string Read(string body) + => Script(body) ?? "[]"; + + #region What the page sent and was told + + /// Every request the page made, oldest first. + public IReadOnlyList Requests() + => JsonSerializer.Deserialize>( + Read("return JSON.stringify(" + + "(window.__51dTest && window.__51dTest.requests) || []);"), + Json) ?? new List(); + + /// + /// The client script's own requests, being the ones to the endpoint it + /// posts its evidence to. + /// + public IReadOnlyList ClientRequests() + => Requests() + .Where(r => r.Url.Contains( + "/api/v4/json", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + /// + /// The preference platform's calls to the shared store, being the read + /// on load and the write when a visitor agrees to share. + /// + public IReadOnlyList SharedStoreRequests() + => Requests() + .Where(r => r.Url.Contains( + "/api/v4/pmp/pref", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + /// Every console line, in the order they were written. + public IReadOnlyList Console() + => JsonSerializer.Deserialize>( + Read("return JSON.stringify(" + + "(window.__51dTest && window.__51dTest.console) || []);"), + Json) + ?.Select(line => $"[{line.Level}] {line.Text}") + .ToList() + ?? new List(); + + private sealed class ConsoleLine + { + [JsonPropertyName("level")] public string Level { get; set; } = ""; + [JsonPropertyName("text")] public string Text { get; set; } = ""; + } + + /// Console lines carrying the given text. + public IReadOnlyList ConsoleMatching(string text) + => Console() + .Where(line => line.Contains(text, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + /// The preferences the platform's action URL was fired with. + public IReadOnlyList Actions() + => JsonSerializer.Deserialize>( + Read("return JSON.stringify(" + + "(window.__51dTest && window.__51dTest.actions) || []);"), + Json) ?? new List(); + + /// Whether the alternative button's own action fired. + public bool AlternativeFired() + => Script( + "return !!(window.__51dTest && window.__51dTest.altFired);"); + + /// + /// The identifiers handed to a change handler the page registered, + /// oldest first. + /// + public IReadOnlyList ChangeIdentifiers() + => JsonSerializer.Deserialize?>>( + Read("return JSON.stringify(" + + "(window.__51dTest && window.__51dTest.changes) || []);"), + Json) + ?.Select(section => + section != null + && section.TryGetValue("idprobglobal", out var value) + ? value ?? "" + : "") + .ToList() + ?? new List(); + + /// Every key the page has in its own local storage. + public IReadOnlyList LocalStorageKeys() + => JsonSerializer.Deserialize>( + Read(@" + var keys = []; + try { + for (var i = 0; i < localStorage.length; i++) { + keys.push(localStorage.key(i)); + } + } catch (e) { } + return JSON.stringify(keys);"), + Json) ?? new List(); + + #endregion + + #region The client script's object + + /// Whether the client script's object exists on the page. + public bool HasClientObject(string objectName = Pages.DefaultObjectName) + => Script( + $"return typeof window['{objectName}'] === 'object'" + + $" && window['{objectName}'] !== null;"); + + /// + /// The global identifier the client script is holding, or an empty + /// string when it holds none. An identifier is created only once an + /// answer has reached the cloud, so an empty string is the normal + /// state of a first request that carried no answer. + /// + public string Identifier(string objectName = Pages.DefaultObjectName) + => Script( + $@"var o = window['{objectName}']; + return (o && o.fodid && o.fodid.idprobglobal) || '';") ?? ""; + + /// + /// Whether the object carries the user prompt block's own entry point, + /// which exists only in the new client script. This is the guard's + /// check made again on what the browser actually loaded, rather than + /// on what the endpoint served the test host. + /// + public bool HasTheNewClientScript( + string objectName = Pages.DefaultObjectName) + => Script( + $"var o = window['{objectName}'];" + + " return !!(o && typeof o.refresh === 'function');"); + + /// + /// The third party cookie result the client script resolved, as the + /// page sees it. Empty where the property has no value. + /// + public string ThirdPartyCookies( + string objectName = Pages.DefaultObjectName) + => Script( + $@"var o = window['{objectName}']; + var d = o && o.device; + var v = d && d.thirdpartycookiesenabled; + return v === undefined || v === null ? '' : String(v);") ?? ""; + + /// + /// The names of the snippet results the client script has stored for + /// this page view, which are what it must send with the request that + /// creates an identifier. Read from the page's own session storage, + /// where the script puts them, so the test does not have to know which + /// properties the resource key happens to carry. + /// + public IReadOnlyList SnippetValueNames( + string objectName = Pages.DefaultObjectName) + => JsonSerializer.Deserialize>( + Read($@" + var prefix = '{objectName}_data_'; + var names = []; + try {{ + for (var i = 0; i < sessionStorage.length; i++) {{ + var key = sessionStorage.key(i); + if (key.indexOf(prefix) === 0) {{ + names.push(key.substring(prefix.length)); + }} + }} + }} catch (e) {{ }} + return JSON.stringify(names);"), + Json) ?? new List(); + + /// + /// The record of the last request's inputs the client script keeps, so + /// that a later page view with a different input asks again rather + /// than answering from the cache. It is stored as a plain string, and + /// null where nothing has been stored. + /// + /// The key is found by its ending rather than spelled out, so a + /// difference in naming between the template and this test shows up as + /// a failed assertion about the contents rather than as a test that + /// silently checks nothing. + /// + /// + public string? CacheRecord(string objectName = Pages.DefaultObjectName) + { + var value = Script($@" + try {{ + for (var i = 0; i < sessionStorage.length; i++) {{ + var key = sessionStorage.key(i); + if (key.indexOf('{objectName}') === 0 + && key.indexOf('_inputs') === key.length - 7) {{ + return sessionStorage.getItem(key); + }} + }} + }} catch (e) {{ }} + return null;"); + return string.IsNullOrEmpty(value) ? null : value; + } + + /// + /// Whether a card is showing and how many rounds the client script has + /// finished, read together in one call, so a test can say that one was + /// true while the other had not moved. Two separate reads would leave a + /// gap in which the answer could change, and the ordering is the whole + /// point of the assertion. + /// + public (bool CardVisible, int RoundsDone) Observe(string card) + { + var answer = Script($@" + var root = pmpRoot(); + var el = root + ? root.querySelector('[data-card=""{card}""]') : null; + var showing = visible(el); + var done = 0; + var all = (window.__51dTest && window.__51dTest.requests) || []; + for (var i = 0; i < all.length; i++) {{ + if (all[i].done + && all[i].url.indexOf('/api/v4/json') !== -1) {{ + done++; + }} + }} + return (showing ? '1' : '0') + '|' + done;") ?? "0|0"; + var parts = answer.Split('|'); + return (parts[0] == "1", int.Parse(parts[1], CultureInfo.InvariantCulture)); + } + + /// + /// Whether the visitor is somewhere the regulation applies, as the + /// client script resolved it. Empty where the property has no value, + /// which is what a resource key that does not carry it produces and + /// what every key produces until the change adding it is released. + /// + public string IsGdpr(string objectName = Pages.DefaultObjectName) + => Script( + $@"var o = window['{objectName}']; + var groups = ['derived', 'device']; + for (var i = 0; i < groups.length; i++) {{ + var g = o && o[groups[i]]; + var v = g && g.isgdpr; + if (v !== undefined && v !== null) {{ return String(v); }} + }} + return '';") ?? ""; + + /// + /// Every script element on the page, by source. Used where a test has + /// to see that something added one. + /// + public IReadOnlyList ScriptSources() + => JsonSerializer.Deserialize>( + Read(@" + return JSON.stringify( + Array.prototype.slice + .call(document.querySelectorAll('script')) + .map(function (s) { return s.src || ''; }) + .filter(function (s) { return s !== ''; }));"), + Json) ?? new List(); + + #endregion + + #region The preference platform + + /// Whether the platform has put anything on the page yet. + public bool PlatformLoaded() => Script("return pmpRoot() !== null;"); + + /// Whether a named card is on the page and visible. + public bool CardVisible(string card) + => Script($@" + var root = pmpRoot(); + if (!root) {{ return false; }} + return visible(root.querySelector('[data-card=""{card}""]'));"); + + /// + /// Whether the floating button is showing and no card is, which is + /// what a visitor who has already answered sees. + /// + /// Both templates are put on the page at once and the dialog is hidden + /// by a class on the container rather than being taken away, so this + /// asks what is visible rather than what exists. + /// + /// + /// The button is found by what it does rather than by its class, + /// because the build renames every class in the stylesheet to one or + /// two letters (pmp/dist/class-map.json turns pmp-fab into y and + /// pmp-popup into ag), so a test written against the class names in + /// the source finds nothing in the bundle a publisher is actually + /// served. The data attributes are not renamed. + /// + /// + public bool BubbleOnly() + => Script(@" + var root = pmpRoot(); + if (!root) { return false; } + if (!visible(root.querySelector('[data-action=""open""]'))) { + return false; + } + var cards = root.querySelectorAll('[data-card]'); + for (var i = 0; i < cards.length; i++) { + if (visible(cards[i])) { return false; } + } + return true;"); + + /// + /// What the dialog looks like right now, in one line, for a failure + /// message. A test that says only that something was not visible + /// leaves the next person to open a browser by hand and find out why. + /// + public string PlatformState() + => Script(@" + var root = pmpRoot(); + if (!root) { return 'the platform has rendered nothing'; } + function describe(el, name) { + if (!el) { return name + '=absent'; } + var box = el.getBoundingClientRect(); + var style = window.getComputedStyle(el); + return name + '=' + (visible(el) ? 'visible' : 'hidden') + + '(' + Math.round(box.width) + 'x' + Math.round(box.height) + + ' display:' + style.display + + ' visibility:' + style.visibility + + ' opacity:' + style.opacity + ')'; + } + var parts = []; + var container = root.querySelector('.pmp'); + parts.push('container class=' + (container + ? container.className : 'absent')); + parts.push(describe( + root.querySelector('[data-action=""open""]'), 'bubble')); + var cards = root.querySelectorAll('[data-card]'); + for (var i = 0; i < cards.length; i++) { + parts.push(describe( + cards[i], 'card:' + cards[i].getAttribute('data-card'))); + } + return parts.join(', ');") ?? "unreadable"; + + /// Presses one of the dialog's buttons. + public void Press(string action) + { + var pressed = Script($@" + var root = pmpRoot(); + if (!root) {{ return false; }} + var el = root.querySelector('[data-action=""{action}""]'); + if (!el) {{ return false; }} + el.click(); + return true;"); + Assert.IsTrue( + pressed, + $"'{action}' was not on the page to press in {BrowserName}. " + + $"The console said: {string.Join(" | ", Console())}"); + } + + /// + /// The answer the platform is holding, through the getter it exposes + /// for a publisher. Empty where it holds none. + /// + public string PlatformPreference() + => Script(@" + var api = window.__51d_pmp; + if (!api || typeof api.preference !== 'function') { return ''; } + var value = api.preference(); + return value === undefined || value === null ? '' : String(value);") + ?? ""; + + /// Opens the dialog again, as a publisher's own link would. + public void OpenPlatform() + => Script("window.__51d_pmp.open(); return null;"); + + /// + /// What the framework surface answers a ping with, being whether the + /// regulation is said to apply. + /// + public bool GdprApplies() + { + Script(@" + window.__51dPing = null; + window.__tcfapi('ping', 2, function (result) { + window.__51dPing = result; + }); + return null;"); + Harness.Until( + () => Script("return window.__51dPing !== null;"), + "the framework surface answered a ping"); + return Script("return !!window.__51dPing.gdprApplies;"); + } + + #endregion + + #region Waiting + + /// + /// Waits until the client script has finished the given number of + /// rounds. A round is one request to the cloud and the response to it, + /// so this is an ordering and not a clock. + /// + public void WaitForClientRounds(int rounds) + => Harness.Until( + () => ClientRequests().Count(r => r.Done) >= rounds, + $"the client script finished {rounds} round(s) in " + + $"{BrowserName}. It finished " + + $"{ClientRequests().Count(r => r.Done)}. {ServedSoFar()}. " + + $"The console said: {string.Join(" | ", Console())}"); + + /// + /// What the browser asked the publisher's website for, which is the + /// one thing that separates a page that never loaded from a page that + /// loaded and then did nothing. + /// + public string ServedSoFar() + => Server.Requests.Count == 0 + ? "the browser asked this website for nothing at all" + : "the website served " + string.Join(", ", Server.Requests); + + /// Waits until the platform has put its dialog on the page. + public void WaitForPlatform() + => Harness.Until( + PlatformLoaded, + $"the preference platform loaded in {BrowserName}. " + + $"{ServedSoFar()}. The console said: " + + $"{string.Join(" | ", Console())}"); + + /// Waits until a named card is showing. + public void WaitForCard(string card) + => Harness.Until( + () => CardVisible(card), + $"the {card} card appeared in {BrowserName}. " + + $"{ServedSoFar()}. The console said: " + + $"{string.Join(" | ", Console())}"); + + /// Waits until the client script's object exists. + public void WaitForClientObject( + string objectName = Pages.DefaultObjectName) + => Harness.Until( + () => HasClientObject(objectName), + $"an object called '{objectName}' appeared in {BrowserName}. " + + $"{ServedSoFar()}. The console said: " + + $"{string.Join(" | ", Console())}"); + + #endregion + + public void Dispose() + { + try + { + Driver.Quit(); + } + catch (Exception) + { + // A browser that has already gone is not a test failure. + } + Driver.Dispose(); + } +} diff --git a/SeleniumApiTests.csproj b/SeleniumApiTests.csproj index c12e1fd..6b05af1 100644 --- a/SeleniumApiTests.csproj +++ b/SeleniumApiTests.csproj @@ -18,6 +18,15 @@ + + From 6f07e304cf57bfc1e9e7922f9fa9a24217b1c7b0 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 15 Sep 2026 06:32:51 +0100 Subject: [PATCH 2/7] TEST: Drive the preference management demo, not synthetic pages The 51Did browser acceptance tests must drive the demo where the Preference Management Platform lives, with every page and every piece of state the tests need placed in that demo, so that each language mirrors the demo and these same tests prove every one of them. The dotnet demo is Examples/Cloud/PreferenceManagement-Web in device-detection-dotnet-examples. - Pages.cs and PageServer.cs are gone. Each test loads a demo route under /cloud/, being common, common-script-first, change, two/one, two/two, consent, no-platform, platform-only or named-object, as site-a.localtest or site-b.localtest on the demo's port. The routes are named once in Demo.cs. - Examples/ExampleApps.cs gains ExampleApps.Demos, with a dotnet entry following the GettingStarted-Web one, and TryCreateDemo. A demo is chosen apart from the Contract tests' example, with DEMO_URL for one already running and DEMO_LANG for one to launch, dotnet unless named, because the two are different apps. The demo is started with exactly 51DEGREES_RESOURCE_KEY and 51DEGREES_CLOUD_ENDPOINT, the input variables every language's demo reads, and ASPNETCORE_URLS for its port. TestsCommon/TestConfig.cs reads those two names, and CLOUD_ROOT_URL keeps its meaning for the rest of the suite. - The path the client script posts to comes from the demo's mode, being the cloud's /api/v4/json on /cloud/ pages and the descriptor's JsonEndpointPath on /pipeline/ pages, chosen with DEMO_MODE. - Every assertion is kept. The consent test gains one, that the string reaching the cloud is exactly TcString.Personalized(), because every language's stub consent platform has to deliver that same string. - Browser51DidTestAttribute replaces [TestMethod] on all fifteen tests and takes the resource key out of every result before the runner sees it, being the failure message, the stack trace and all output. The base class fails any test not marked with it, because this suite is public and a failure message routinely quotes a script source naming the key. - The tests that create a 51Did for a standard or personalized answer need a resource key whose products include CloudV5FODiD. The harness asks the service first and skips with the service's own reason where the key cannot create. - The shared choice skips only where the cloud is plain HTTP and not localhost, because both browsers kept its Secure SameSite=None cookie from http://localhost:5050 when measured. The share card ordering is counted inside the page. The common path asserts that the share card appears before any further client round finishes after the visitor answers. It used to read the finished count from the test before the press and then poll the page every 100 ms, and that failed 3 times in 7 runs before the move and once in 10 over HTTPS against the demo. A timeline recorded inside the page caught that failure: 0ms recording started | 1ms request 3 seen already finished | 1ms request 3 finished | 291ms pressed standard | 302ms request 4 seen carrying an answer | 302ms share card in the dialog | 302ms share card visible | 500ms request 4 finished with the assertion reporting Expected:<1>. Actual:<2>. The card was up 198 ms before the answered round finished, so nothing waited, and the test's first poll that saw the card came after 500 ms. The two counts are now taken inside the page, at the moment of the press and at the first animation frame in which the card is visible, and the assertion is still that they are equal. It still fails where the platform waits, because a round finishing before the card becomes visible is counted first within a frame, and the failure message carries the whole timeline. Against a locally run cloud, with the demo launched by the suite, the category gave 14 passed and 1 skipped, the skip being the IsGdpr test that cannot pass until that property is released. --- Browser51Did/Browser51DidTestBase.cs | 136 ++++++-- .../ConsentPlatformAcceptanceTests.cs | 31 +- Browser51Did/Demo.cs | 278 +++++++++++++++ Browser51Did/Harness.cs | 319 +++++++++++------ Browser51Did/IsGdprAcceptanceTests.cs | 24 +- Browser51Did/PageServer.cs | 193 ----------- Browser51Did/Pages.cs | 324 ------------------ Browser51Did/PlatformAcceptanceTests.cs | 129 +++---- Browser51Did/PlatformAddsClientScriptTests.cs | 46 +-- Browser51Did/README.md | 119 +++++++ Browser51Did/TcString.cs | 15 +- Browser51Did/Visitor.cs | 216 +++++++++--- Examples/ExampleApps.cs | 79 +++++ README.md | 3 + TestsCommon/TestConfig.cs | 8 + 15 files changed, 1058 insertions(+), 862 deletions(-) create mode 100644 Browser51Did/Demo.cs delete mode 100644 Browser51Did/PageServer.cs delete mode 100644 Browser51Did/Pages.cs create mode 100644 Browser51Did/README.md diff --git a/Browser51Did/Browser51DidTestBase.cs b/Browser51Did/Browser51DidTestBase.cs index 91389da..17dc049 100644 --- a/Browser51Did/Browser51DidTestBase.cs +++ b/Browser51Did/Browser51DidTestBase.cs @@ -1,20 +1,102 @@ #nullable enable using System; -using System.Collections.Generic; -using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; using FiftyOne.Did.Model; +using Microsoft.VisualStudio.TestTools.UnitTesting; namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; +/// +/// A test method whose every result has the resource key taken out of it +/// before the runner sees it, being the failure message, the stack trace +/// and everything the test wrote. +/// +/// This suite is public and its CI log is public, and a failure message +/// here routinely quotes what a page did, which includes the client +/// script's address, and that address names the resource key. Redacting +/// each message by hand leaves the next message written to leak it, so it +/// is done once, here, for every result. +/// refuses to run a test that is not marked with this attribute. +/// +/// +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +public sealed class Browser51DidTestAttribute : TestMethodAttribute +{ + /// Passes the declaring file and line on, as MSTest needs. + public Browser51DidTestAttribute( + [CallerFilePath] string callerFilePath = "", + [CallerLineNumber] int callerLineNumber = -1) + : base(callerFilePath, callerLineNumber) + { + } + + /// + public override async Task ExecuteAsync( + ITestMethod testMethod) + { + var results = await base.ExecuteAsync(testMethod) + .ConfigureAwait(false); + foreach (var result in results) + { + Redact(result); + } + return results; + } + + private static void Redact(TestResult result) + { + result.LogOutput = RedactedOrNull(result.LogOutput); + result.LogError = RedactedOrNull(result.LogError); + result.DebugTrace = RedactedOrNull(result.DebugTrace); + result.TestContextMessages = RedactedOrNull(result.TestContextMessages); + var failure = result.TestFailureException; + if (failure is null || string.IsNullOrEmpty(Harness.Resource)) + { + return; + } + var whole = failure.ToString(); + if (whole.Contains(Harness.Resource, StringComparison.Ordinal) == false) + { + return; + } + // The exception cannot be edited, so a new one carries the redacted + // text. Its own stack trace would point here, so the original one is + // kept in the message, where the line that failed can still be read. + var text = Harness.Redacted( + $"{InnermostMessage(failure)}\nWhere it failed, kept because " + + $"the message was redacted:\n{whole}"); + result.TestFailureException = + result.Outcome == UnitTestOutcome.Inconclusive + ? new AssertInconclusiveException(text) + : new AssertFailedException(text); + } + + private static string InnermostMessage(Exception failure) + { + var inner = failure; + while (inner.InnerException is not null) + { + inner = inner.InnerException; + } + return inner.Message; + } + + private static string? RedactedOrNull(string? text) + => text is null ? null : Harness.Redacted(text); +} + /// /// What every browser acceptance test class shares, being the guard, the -/// skip when the harness is not configured, and the reading of a 51Did. +/// skip when the harness is not configured, the demo, and the reading of a +/// 51Did. /// /// The guard runs once for each class from the set up, not from inside a -/// test, so that a container built against the old client script fails -/// every test in the class rather than letting one of them pass for the -/// wrong reason. +/// test, so that a cloud built against the old client script fails every +/// test in the class rather than letting one of them pass for the wrong +/// reason. /// /// public abstract class Browser51DidTestBase @@ -29,9 +111,12 @@ public abstract class Browser51DidTestBase /// The second browser, for the same reason. protected const string Firefox = "Firefox"; + /// Set by MSTest, and used to find the running test. + public TestContext TestContext { get; set; } = null!; + /// - /// Fails every test in the class unless the client script served by - /// the endpoint under test is the new one. See + /// Fails every test in the class unless the client script the demo's + /// pages load is the new one. See /// . /// [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] @@ -49,29 +134,29 @@ public static void GuardTheClientScript(TestContext context) [TestInitialize] public void RequireHarness() { + var method = GetType().GetMethod(TestContext.TestName ?? string.Empty); + if (method?.GetCustomAttribute() is null) + { + Assert.Fail( + $"{TestContext.TestName} is not marked [Browser51DidTest], " + + "so a failure in it could print the resource key into a " + + "public log. Mark it [Browser51DidTest] instead of " + + "[TestMethod]."); + } if (Harness.Configured == false) { - Assert.Inconclusive( - "The browser acceptance harness is not configured. Set " - + "FIFTYONE_CONTEXT_SELENIUM_BASEURL and " - + "FIFTYONE_CONTEXT_SELENIUM_RESOURCE, or run ci/test.ps1 " - + "with -Browser51Did $true, which starts the container and " - + "sets them. See README.md."); + Assert.Inconclusive(Harness.NotConfiguredReason); } + Demo.Chosen.EnsureStarted(); } /// - /// A browser looking at a website served for the length of this test. + /// A browser, which will load the demo's pages as the two publisher + /// sites. /// - protected static Visitor NewVisitor( - string browser, - PageServer server, - IReadOnlyDictionary? preferences = null) + protected static Visitor NewVisitor(string browser) => new( - browser == Firefox - ? Harness.NewFirefox(preferences) - : Harness.NewChrome(preferences), - server, + browser == Firefox ? Harness.NewFirefox() : Harness.NewChrome(), browser); /// @@ -85,9 +170,8 @@ protected static Visitor NewVisitor( /// /// Refuses to go on where this resource key cannot create an /// identifier for a standard or personalized answer, which needs a - /// licence carrying the CloudV5FODiD product. See - /// for why a throwaway record cannot - /// have one. + /// resource key whose products include CloudV5FODiD. The service is + /// asked, and the skip carries its own reason. /// protected static void RequireMarketingIdentifiers() => Harness.RequireMarketingIdentifiers(); diff --git a/Browser51Did/ConsentPlatformAcceptanceTests.cs b/Browser51Did/ConsentPlatformAcceptanceTests.cs index fd5e3e9..49e696c 100644 --- a/Browser51Did/ConsentPlatformAcceptanceTests.cs +++ b/Browser51Did/ConsentPlatformAcceptanceTests.cs @@ -32,18 +32,12 @@ public class ConsentPlatformAcceptanceTests : Browser51DidTestBase /// case, and the identifier that comes back records that the usage was /// decoded rather than stated. /// - [TestMethod] + [Browser51DidTest] public void ConsentPlatformOnly_SecondRequestCarriesTheString() { - using var server = new PageServer(); RequireMarketingIdentifiers(); - server.Put("/", Pages.Page( - "Consent platform only", - Pages.ConsentPlatform(TcString.Personalized()), - Pages.ClientScriptTag())); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.Consent); visitor.WaitForClientRounds(1); Assert.IsTrue( visitor.HasTheNewClientScript(), @@ -83,6 +77,15 @@ public void ConsentPlatformOnly_SecondRequestCarriesTheString() second.Form("tcstring"), "the framework string the platform delivered must reach the " + $"cloud. The body was: {second.Body}"); + // The demo's stub consent platform delivers one fixed string, and + // every language's demo has to deliver the same one, so it is + // named here rather than trusted. + Assert.AreEqual( + TcString.Personalized(), + second.Form("tcstring"), + "the stub consent platform on the demo's consent page must " + + "deliver the string granting every purpose, built by " + + $"TcString.Personalized(). The body was: {second.Body}"); Assert.IsNull( second.Form("id.usage"), "the client script must not decide the usage itself. A string " @@ -116,15 +119,11 @@ public void ConsentPlatformOnly_SecondRequestCarriesTheString() /// so a later page view with an answer is a different input and asks /// again rather than reusing this one. /// - [TestMethod] + [Browser51DidTest] public void NoPlatformAtAll_NoIdentifierAndOneWarning() { - using var server = new PageServer(); - server.Put("/", Pages.Page( - "No platform", Pages.ClientScriptTag())); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.NoPlatform); visitor.WaitForClientRounds(1); Assert.IsTrue( visitor.HasTheNewClientScript(), diff --git a/Browser51Did/Demo.cs b/Browser51Did/Demo.cs new file mode 100644 index 0000000..c11cd40 --- /dev/null +++ b/Browser51Did/Demo.cs @@ -0,0 +1,278 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Examples; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// The routes every language's demo serves, as the dotnet demo in +/// device-detection-dotnet-examples defines them. A test names the page it +/// loads by one of these and nothing else, so the same test drives any +/// demo that serves the same pages. +/// +public static class Routes +{ + /// The platform, then the client script. + public const string Common = "common"; + + /// The client script, then the platform. + public const string CommonScriptFirst = "common-script-first"; + + /// The platform, the client script and the change watcher. + public const string Change = "change"; + + /// The first of two pages carrying what does. + public const string TwoOne = "two/one"; + + /// The second of those two pages. + public const string TwoTwo = "two/two"; + + /// The stub consent platform, then the client script. + public const string Consent = "consent"; + + /// The client script alone. + public const string NoPlatform = "no-platform"; + + /// The platform alone, with no client script tag. + public const string PlatformOnly = "platform-only"; + + /// + /// The platform alone, naming the client script's object + /// . + /// + public const string NamedObject = "named-object"; +} + +/// +/// How a demo's pages get their client script, being the route prefix the +/// demo serves them under. +/// +public sealed class DemoMode +{ + /// + /// Pages that load the client script straight from the cloud, which + /// posts its evidence to the cloud's own endpoint. + /// + public static readonly DemoMode Cloud = new("cloud", null); + + /// + /// Pages whose client script the demo's own pipeline serves, which + /// posts to that pipeline's endpoint, being the path the demo's + /// descriptor names. + /// + public static readonly DemoMode Pipeline = new("pipeline", "descriptor"); + + /// The route prefix, and the name DEMO_MODE takes. + public string Name { get; } + + private readonly string? _pathFromDescriptor; + + private DemoMode(string name, string? pathFromDescriptor) + { + Name = name; + _pathFromDescriptor = pathFromDescriptor; + } + + /// + /// The path the client script posts its evidence to on this mode's + /// pages, which is how a test tells the client script's own requests + /// from everything else a page does. + /// + public string JsonEndpointPath(ExampleDescriptor? descriptor) + => _pathFromDescriptor is null + ? "/api/v4/json" + : descriptor?.JsonEndpointPath + ?? throw new InvalidOperationException( + "The pipeline mode needs a registered demo, because its " + + "descriptor names the path the client script posts to."); +} + +/// +/// The demo under test, started once for the whole run. +/// +/// It is the demo at DEMO_URL where that is set, and otherwise the DEMO_LANG +/// demo, dotnet unless another is named, which the suite launches from its +/// sibling checkout through , the way +/// the Contract tests launch an example. It is started with exactly the two +/// input variables every language's demo reads, 51DEGREES_RESOURCE_KEY and +/// 51DEGREES_CLOUD_ENDPOINT, taken from this run's own environment, and +/// with the runtime's own way of choosing a port. DEMO_MODE chooses the +/// route prefix, cloud unless pipeline is named. +/// +/// +public sealed class Demo +{ + private static readonly Lazy ChosenOnce = + new(Choose, LazyThreadSafetyMode.ExecutionAndPublication); + + /// + /// The demo this run tests. Deciding starts nothing, and + /// starts it. + /// + public static Demo Chosen => ChosenOnce.Value; + + private readonly IExampleApp? _app; + private readonly ExampleDescriptor? _descriptor; + private readonly string? _unavailable; + private readonly object _startLock = new(); + private bool _started; + private string? _startFailure; + + /// Which pages are loaded. + public DemoMode Mode { get; } + + /// The demo's name, for a message. + public string Name { get; } + + private Demo( + IExampleApp? app, + ExampleDescriptor? descriptor, + DemoMode mode, + string name, + string? unavailable) + { + _app = app; + _descriptor = descriptor; + Mode = mode; + Name = name; + _unavailable = unavailable; + } + + private static Demo Choose() + { + var modeName = Environment.GetEnvironmentVariable("DEMO_MODE"); + var mode = string.Equals( + modeName, DemoMode.Pipeline.Name, StringComparison.OrdinalIgnoreCase) + ? DemoMode.Pipeline + : DemoMode.Cloud; + var name = $"the {ExampleApps.SelectedDemoLang} demo's /{mode.Name}/ pages"; + return ExampleApps.TryCreateDemo( + out var app, out var descriptor, out var skipReason) + ? new Demo(app, descriptor, mode, name, null) + : new Demo(null, descriptor, mode, name, skipReason); + } + + /// + /// The path the client script posts its evidence to on the pages being + /// loaded. + /// + public string JsonEndpointPath => Mode.JsonEndpointPath(_descriptor); + + /// + /// Starts the demo where it has not been started, once for the whole + /// run, because launching one costs far more than any test here. A demo + /// that fails to start fails every test with the same reason. + /// + public void EnsureStarted() + { + lock (_startLock) + { + if (_startFailure != null) + { + Assert.Fail(_startFailure); + } + if (_started) + { + return; + } + if (_app is null) + { + _startFailure = $"No demo can be started. {_unavailable}"; + Assert.Fail(_startFailure); + return; + } + try + { + _app.StartAsync( + new ExampleAppOptions( + TestHelpers.GetRandomUnusedPort(), + new Uri(Harness.CloudUrl + "/"), + Harness.Resource!, + new Dictionary()), + CancellationToken.None) + .GetAwaiter().GetResult(); + _started = true; + } + catch (Exception error) + { + // The demo's own output is in the message, and a page it + // logged could carry the resource key. + _startFailure = Harness.Redacted( + $"{Name} could not be started, so no test ran against " + + $"it. {error.Message}"); + Assert.Fail(_startFailure); + } + } + } + + /// + /// Stops the demo where this run started it, and does nothing where no + /// test ever asked for one, so a run of another category is left alone. + /// + public static void StopIfStarted() + { + if (ChosenOnce.IsValueCreated == false) + { + return; + } + var chosen = ChosenOnce.Value; + lock (chosen._startLock) + { + if (chosen._app is null || chosen._started == false) + { + return; + } + chosen._app.DisposeAsync().AsTask().GetAwaiter().GetResult(); + chosen._started = false; + } + } + + /// + /// The address of one of the demo's pages as one of the publisher + /// sites. The cache buster is on every navigation, because a page held + /// in the browser's cache would carry the recorder from an earlier test. + /// + public string PageUrl(string site, string route) + { + EnsureStarted(); + return $"http://{site}:{_app!.BaseUrl.Port}/{Mode.Name}/{route}" + + $"?v={DateTime.UtcNow.Ticks}"; + } + + /// + /// Where the guard fetches the client script from, server to server, + /// being what the pages load. On the cloud pages that is the cloud's + /// script for the resource key, and on the pipeline pages it is the + /// demo's own. + /// + public string ClientScriptFetchUrl() + { + if (Mode == DemoMode.Cloud) + { + return $"{Harness.CloudUrl}/api/v4/{Harness.Resource}.js"; + } + EnsureStarted(); + return new Uri(_app!.BaseUrl, "51Degrees.core.js").ToString(); + } +} + +/// +/// Stops the demo this run started once every test has finished. +/// +[TestClass] +public class Browser51DidRunCleanup +{ + /// + /// MSTest allows one assembly clean up in a test assembly and this is + /// it, so anything else that has to happen at the end of a run belongs + /// in this method rather than beside it. A demo left running would hold + /// its port after the run. + /// + [AssemblyCleanup] + public static void StopTheDemo() => Demo.StopIfStarted(); +} diff --git a/Browser51Did/Harness.cs b/Browser51Did/Harness.cs index 5c35558..9c63491 100644 --- a/Browser51Did/Harness.cs +++ b/Browser51Did/Harness.cs @@ -3,92 +3,133 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Net.Http; using System.Text.Json; using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using FiftyOne.Pipeline.Cloud.Tests.Common; using Microsoft.VisualStudio.TestTools.UnitTesting; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.Firefox; +using OpenQA.Selenium.Remote; namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// /// Everything the browser acceptance tests share, being where the cloud /// is, which resource key to use, the two site names a cross site test -/// needs, the browsers, and the guard that stops the whole class running -/// against the old client script. +/// needs, the browsers, and the guard that stops a test running against +/// the old client script. /// -/// The harness is the same one CrossBrowserContextTests uses, being -/// the container that terminates TLS itself, started by -/// ci/test.ps1 with a throwaway resource key and a throwaway -/// context secret. These tests reuse its environment variables rather -/// than adding a second set. +/// The thing under test is a demo, a web app serving the pages these tests +/// load, which every language mirrors, started with a cloud endpoint and a +/// resource key. The preference platform and the shared store come from +/// that cloud whichever demo is chosen. See . /// /// public static class Harness { /// - /// The cloud instance serving TLS itself, for example - /// https://localhost:8081. Shared with the cross browser context - /// tests, so one container serves both. + /// A setting read through the suite's own , + /// with the message it gives where the variable is missing. /// - public static readonly string? BaseUrl = - Environment.GetEnvironmentVariable( - "FIFTYONE_CONTEXT_SELENIUM_BASEURL"); + private readonly record struct Setting(string? Value, string? Missing); + + private static Setting Read(Func require) + { + try + { + return new Setting(require(), null); + } + catch (InvalidOperationException missing) + { + return new Setting(null, missing.Message); + } + } + + private static readonly Setting Endpoint = + Read(() => TestConfig.Instance().DemoCloudEndpoint); + + private static readonly Setting ResourceKey = + Read(() => TestConfig.Instance().DemoResourceKey); /// - /// The resource key the pages use. - /// - /// It is not the throwaway one the cross browser context tests use. - /// Creating an identifier for a standard or personalized answer needs - /// a licence carrying the CloudV5FODiD product, which - /// DidOnPremiseEngine.TryResolveLicenseId scans the customer's - /// licence keys for, and refusing without one is deliberate. A - /// throwaway record cannot have that product, because the product - /// lives inside a real signed licence key rather than being a name a - /// record can claim, which is why the context tests ask for the - /// non-marketing usage. Everything the acceptance tests are for is a - /// marketing usage, so they use the 51Did entitled resource key that - /// is already committed in - /// host/FiftyOne.Pipeline.Cloud.Tests.Common/testConfig.json - /// as fodid_resource_key and that ci/test.ps1 already uses for - /// the identifier surface check. That customer's record also carries - /// a licence key, which is what the shared store needs. - /// - /// - /// It falls back to the context harness key so that a developer who - /// sets only the two context variables gets a run that says plainly - /// what it could not do rather than one that will not start. - /// + /// The cloud, for example https://localhost:5443, with no trailing + /// slash, taken from 51DEGREES_CLOUD_ENDPOINT, the variable the demo is + /// started with. That endpoint names the api/v4 path, as every reader of + /// it expects, and the path is taken off here, because what the tests + /// ask the cloud directly they build from its root. An endpoint written + /// without the path is taken as the root. /// - public static readonly string? Resource = - FirstSet( - "FIFTYONE_BROWSER51DID_RESOURCE", - "FIFTYONE_CONTEXT_SELENIUM_RESOURCE"); - - private static string? FirstSet(params string[] names) + public static string? CloudUrl { - foreach (var name in names) + get { - var value = Environment.GetEnvironmentVariable(name); - if (string.IsNullOrEmpty(value) == false) + if (string.IsNullOrEmpty(Endpoint.Value)) { - return value; + return null; } + var root = Endpoint.Value.TrimEnd('/'); + const string ApiPath = "/api/v4"; + return root.EndsWith(ApiPath, StringComparison.OrdinalIgnoreCase) + ? root.Substring(0, root.Length - ApiPath.Length) + : root; } - return null; } + /// + /// The resource key the demo is started with, from + /// 51DEGREES_RESOURCE_KEY, the variable every language's demo reads. + /// + /// The tests that create a 51Did for a standard or personalized answer + /// need a resource key whose products include CloudV5FODiD. + /// asks the service first, + /// and where the key cannot create, those tests skip with the service's + /// own reason. + /// + /// + public static string? Resource => ResourceKey.Value; + /// /// Whether the harness is configured at all. Unset means a developer - /// ran the project on its own, so the tests report inconclusive rather - /// than failing, exactly as the context tests do. + /// ran the category without saying where the cloud is or which key to + /// use, so the tests report inconclusive rather than failing, the way + /// the suite's other tests do when their configuration is missing. /// public static bool Configured => - string.IsNullOrEmpty(BaseUrl) == false + string.IsNullOrEmpty(CloudUrl) == false && string.IsNullOrEmpty(Resource) == false; + /// Why the harness is not configured, for the skip. + public static string NotConfiguredReason => + "The 51Did browser acceptance tests are not configured. " + + string.Join( + " ", + new[] + { + string.IsNullOrEmpty(CloudUrl) ? Endpoint.Missing : null, + string.IsNullOrEmpty(Resource) ? ResourceKey.Missing : null, + }.Where(missing => missing != null)) + + " 51DEGREES_CLOUD_ENDPOINT names the cloud including its api/v4 " + + "path, and 51DEGREES_RESOURCE_KEY a resource key the cloud " + + "creates 51Dids for standard and personalized answers with. Both " + + "are handed to the demo unchanged. See Browser51Did/README.md."; + + /// + /// The object name a page uses when it does not ask for another one, + /// which is what the client script and the preference platform both + /// fall back to. + /// + public const string DefaultObjectName = "fod"; + + /// + /// The object name the demo's page + /// gives the preference platform in data-object-name. + /// + public const string NamedObject = "fiftyOneData"; + /// /// The first publisher site. It is a name and not localhost, because /// the cloud is on localhost and a page there would be the same site @@ -126,8 +167,9 @@ public static class Harness /// Text that exists only in the client script's new user prompt block, /// being the message it logs once a page view has reached the server's /// maximum number of rounds. The guard below refuses to let any test - /// in this namespace run unless the script served by the endpoint under - /// test carries it, which is what makes a green run mean something. + /// in this namespace run unless the script served by the implementation + /// under test carries it, which is what makes a green run mean + /// something. /// /// Settled by the template work package. If that wording changes, this /// is the one place to change it, and the report for that package @@ -189,24 +231,12 @@ public static class Harness #endregion - /// - /// The client script for the throwaway resource key, as the page asks - /// for it. - /// - public static string ClientScriptUrl(string? objectName = null) - { - var url = $"{BaseUrl}/api/v4/{Resource}.js"; - return objectName is null - ? url - : $"{url}?fod-js-object-name={objectName}"; - } - /// The preference platform's loader, as the page asks for it. - public static string PlatformLoaderUrl() => $"{BaseUrl}/api/v4/pmp"; + public static string PlatformLoaderUrl() => $"{CloudUrl}/api/v4/pmp"; /// /// Server to server, with certificate validation relaxed because the - /// container under test serves a certificate issued for another name. + /// cloud under test may serve a certificate issued for another name. /// The browsers are told to accept it for the same reason. /// private static readonly HttpClient Reader = new( @@ -224,9 +254,9 @@ public static string ClientScriptUrl(string? objectName = null) private static string? _guardFailure; /// - /// The guard. Fetches the client script from the endpoint the tests are - /// about to drive a browser at and refuses to go on unless the body - /// carries both of the markers below. + /// The guard. Fetches the client script the demo's pages load + /// before the tests drive a browser at them, and refuses to go on unless + /// the body carries both of the markers below. /// /// Two markers, because they say different things. The iteration limit /// message says the template is the new one. It sits outside the user @@ -238,11 +268,11 @@ public static string ClientScriptUrl(string? objectName = null) /// off, which passes nothing it is meant to prove. /// /// - /// It runs once per class from the set up rather than inside a test, so - /// no test here can run against the old script and report green. A run - /// against a cloud built on the released 4.5.104 package fails every + /// It runs once per class from the set up rather than inside a test, + /// so no test here can run against the old script and report green. A + /// run against a cloud built on the released 4.5.104 package fails every /// test in the class with the line below, which says what was served - /// and what was wanted. + /// and what was wanted. The answer is kept, so it is asked once a run. /// /// public static void RequireTheNewClientScript() @@ -258,8 +288,9 @@ public static void RequireTheNewClientScript() Console.WriteLine(_guardEvidence); return; } + var implementation = Demo.Chosen; + var url = implementation.ClientScriptFetchUrl(); string body; - var url = ClientScriptUrl(); try { body = Reader.GetStringAsync(url).Result; @@ -269,8 +300,9 @@ public static void RequireTheNewClientScript() _guardFailure = "The client script could not be fetched from " + $"{Redacted(url)}, " - + "so there is no way to tell which template the " - + $"container was built from. {error.Message}"; + + "so there is no way to tell which template " + + $"{implementation.Name} was built from. " + + Redacted(error.Message); Assert.Fail(_guardFailure); return; } @@ -282,14 +314,12 @@ public static void RequireTheNewClientScript() { _guardFailure = "The client script served by " - + $"{Redacted(url)} does not carry " - + $"'{marker}', which is what says {says}. Every " - + "test in this class would be proving the wrong " - + "thing, so none of them runs. Move the " - + "FiftyOne.Pipeline.JavaScriptBuilder pin to the " - + "package built from the new template and build " - + $"the image again. The script was {body.Length} " - + "bytes."; + + $"{implementation.Name} at {Redacted(url)} does " + + $"not carry '{marker}', which is what says " + + $"{says}. Every test would be proving the wrong " + + "thing, so none of them runs. " + + Remedy(implementation) + + $" The script was {body.Length} bytes."; Assert.Fail(_guardFailure); return; } @@ -297,13 +327,24 @@ public static void RequireTheNewClientScript() $"'{marker}' at {found}: {Around(body, found)}"); } _guardEvidence = - "Client script guard passed. " + $"Client script guard passed for {implementation.Name}. " + $"{Redacted(url)} served {body.Length} " + "bytes carrying " + string.Join(" and ", evidence); Console.WriteLine(_guardEvidence); } } + /// What to do about a client script without the markers. + private static string Remedy(Demo demo) + => demo.Mode == DemoMode.Cloud + ? "Move the cloud's FiftyOne.Pipeline.JavaScriptBuilder pin to " + + "the package built from the new template and build the cloud " + + "again." + : $"Build {demo.Name} against a JavaScript builder carrying the " + + "new template, with a 51Did element in its pipeline, which " + + "is what makes the user prompt block render, and start it " + + "again."; + /// /// What the served client script has to carry, and what each one /// proves. See for why one is @@ -355,10 +396,12 @@ private static readonly (string Marker, string Says)[] RequiredMarkers = /// identifier for a marketing answer, which is what standard and /// personalized both are. /// - /// The service is asked rather than a flag being read, so the reason - /// the test reports is the service's own words rather than a guess, - /// and a key that stops carrying the product later says so instead of - /// failing somewhere in a browser. + /// The cloud is asked rather than a flag being read, so the reason the + /// test reports is the service's own words rather than a guess, and a + /// key that stops carrying the product later says so instead of + /// failing somewhere in a browser. It is the cloud that is asked + /// whichever implementation is under test, because an example creates + /// nothing itself and passes the request on to the cloud. /// /// public static void RequireMarketingIdentifiers() @@ -378,21 +421,19 @@ public static void RequireMarketingIdentifiers() + "marketing answer, which is what standard and " + "personalized both are, so nothing here would be " + "testing the thing it is for. The service said: " - + _marketingRefusal - + " Point FIFTYONE_BROWSER51DID_RESOURCE at a resource key " - + "whose customer holds a licence carrying the CloudV5FODiD " - + "product, such as fodid_resource_key in " - + "host/FiftyOne.Pipeline.Cloud.Tests.Common/testConfig.json."); + + Redacted(_marketingRefusal) + + " Point 51DEGREES_RESOURCE_KEY at a resource key " + + "the service creates standard identifiers for."); } } /// - /// Asks the service to make a standard identifier and reports why it + /// Asks the cloud to make a standard identifier and reports why it /// would not, or null where it did. /// private static string? MarketingRefusal() { - var url = $"{BaseUrl}/api/v4/json?resource={Resource}" + var url = $"{CloudUrl}/api/v4/json?resource={Resource}" + "&id.usage=standard&values=FODiD.IdProbGlobal"; string body; try @@ -455,7 +496,7 @@ public static void RequireSharedStore() "The cloud will not hold a choice for this resource key, " + "so a choice cannot be carried between sites and nothing " + "here would be testing that it is. The service said: " - + _sharedStoreRefusal); + + Redacted(_sharedStoreRefusal)); } } @@ -470,7 +511,7 @@ public static void RequireSharedStore() try { using var request = new HttpRequestMessage( - HttpMethod.Post, $"{BaseUrl}/api/v4/pmp/pref") + HttpMethod.Post, $"{CloudUrl}/api/v4/pmp/pref") { Content = new FormUrlEncodedContent( new Dictionary @@ -495,6 +536,42 @@ public static void RequireSharedStore() } } + /// + /// Refuses to go on where a browser would not keep the shared choice's + /// cookie at all, so a choice could not be carried between sites + /// whatever the code did. + /// + /// The cloud sets that cookie Secure and SameSite=None, and a browser + /// keeps such a cookie only from a secure origin. HTTPS is one, and so + /// is localhost, which browsers treat as secure whatever the scheme. + /// Measured on 15 September 2026 against a service on + /// http://localhost:5050, both browsers kept the cookie and both tests + /// that call this passed. So what is refused is a cloud reached over + /// plain HTTP by any other name, and the suite's usual target, + /// http://localhost:8080, is not refused. Other loopback names such as + /// 127.0.0.1 were not measured, so they are refused rather than + /// assumed. + /// + /// + public static void RequireSecureCookies() + { + var cloud = new Uri(CloudUrl!); + if (cloud.Scheme == Uri.UriSchemeHttps + || string.Equals( + cloud.Host, "localhost", StringComparison.OrdinalIgnoreCase)) + { + return; + } + Assert.Inconclusive( + $"The cloud is at {Redacted(CloudUrl!)}, which is plain HTTP " + + "and not localhost, so a browser will not keep the shared " + + "choice's cookie, which the cloud sets Secure and " + + "SameSite=None. A choice cannot be carried between sites " + + "there, so nothing here would be testing that it is. Point " + + "51DEGREES_CLOUD_ENDPOINT at the cloud's HTTPS listener to run " + + "this."); + } + /// /// The group of sites a choice is shared across in these tests. The /// name is what the cookie holding the choice is derived from, so the @@ -506,15 +583,16 @@ private static string Truncate(string value) => value.Length <= 300 ? value : value.Substring(0, 300) + "..."; /// - /// A URL with the resource key taken out of it, for anything a test + /// Text with the resource key taken out of it, for anything a test /// prints. A failure message and the guard's evidence both end up in a - /// run log and in a pull request body, and a resource key is a - /// credential that must never be written down anywhere. + /// run log and in a pull request body, this suite runs in public + /// repositories' builds, and a resource key is a credential that must + /// never be written down anywhere. /// - public static string Redacted(string url) + public static string Redacted(string text) => string.IsNullOrEmpty(Resource) - ? url - : url.Replace(Resource, "", StringComparison.Ordinal); + ? text + : text.Replace(Resource, "", StringComparison.Ordinal); /// One line of the script either side of the match. private static string Around(string body, int at) @@ -558,7 +636,7 @@ public static IWebDriver NewChrome( preference.Key, preference.Value); } } - return new ChromeDriver(options); + return Start(options); } /// @@ -598,12 +676,35 @@ public static IWebDriver NewFirefox( preference.Key, Convert.ToString( preference.Value, - CultureInfo.InvariantCulture)); + CultureInfo.InvariantCulture) ?? string.Empty); break; } } } - return new FirefoxDriver(options); + return Start(options); + } + + /// + /// A local browser, or one on the Selenium grid SELENIUM_URL names, + /// which is how every browser in this suite is started. A grid has to + /// share this machine's network, because the site names resolve to + /// 127.0.0.1 inside the browser. + /// + private static IWebDriver Start(DriverOptions options) + { + if (ExternalSeleniumHelper.IsExternalSelenium(out var seleniumUrl)) + { + ExternalSeleniumHelper.AddExternalSeleniumArguments(options); + return new RemoteWebDriver(new Uri(seleniumUrl), options); + } + return options switch + { + ChromeOptions chrome => new ChromeDriver(chrome), + FirefoxOptions firefox => new FirefoxDriver(firefox), + _ => throw new ArgumentOutOfRangeException( + nameof(options), + $"{options.GetType().Name} is not a browser these tests use."), + }; } /// @@ -623,9 +724,9 @@ public static void Until( } Thread.Sleep(100); } - Assert.Fail( + Assert.Fail(Redacted( $"Waited {Patience.TotalSeconds:0} seconds and " - + $"{whatWasWaitedFor} never happened."); + + $"{whatWasWaitedFor} never happened.")); } #endregion diff --git a/Browser51Did/IsGdprAcceptanceTests.cs b/Browser51Did/IsGdprAcceptanceTests.cs index 35f6770..be036a2 100644 --- a/Browser51Did/IsGdprAcceptanceTests.cs +++ b/Browser51Did/IsGdprAcceptanceTests.cs @@ -32,17 +32,11 @@ public class IsGdprAcceptanceTests : Browser51DidTestBase /// surface reports it, and a false value does not stop the visitor /// being asked. /// - [TestMethod] + [Browser51DidTest] public void IsGdpr_ReadFromTheClientScript_SetsGdprApplies() { - using var server = new PageServer(); - server.Put("/", Pages.Page( - "Is the regulation in force", - Pages.PlatformTag(new Pages.PlatformSettings()), - Pages.ClientScriptTag())); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.Common); visitor.WaitForPlatform(); visitor.WaitForClientRounds(1); @@ -85,17 +79,11 @@ public void IsGdpr_ReadFromTheClientScript_SetsGdprApplies() /// With no value to read, the platform says so once and carries on as /// though the regulation applies, which is the safe way round. /// - [TestMethod] + [Browser51DidTest] public void IsGdprAbsent_PlatformWarnsAndAssumesItApplies() { - using var server = new PageServer(); - server.Put("/", Pages.Page( - "No isgdpr", - Pages.PlatformTag(new Pages.PlatformSettings()), - Pages.ClientScriptTag())); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.Common); visitor.WaitForPlatform(); visitor.WaitForClientRounds(1); diff --git a/Browser51Did/PageServer.cs b/Browser51Did/PageServer.cs deleted file mode 100644 index c00c9c5..0000000 --- a/Browser51Did/PageServer.cs +++ /dev/null @@ -1,193 +0,0 @@ -#nullable enable - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Net.Sockets; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; - -/// -/// The publisher's website, for the length of one test. It serves the page -/// fixtures and nothing else. -/// -/// It is deliberately not a proxy for the cloud. The pages load the client -/// script and the preference platform straight from the cloud's own origin, -/// which is what makes the cloud a third party to the page and the shared -/// cookie a third party cookie. A proxy would put both on one origin and -/// the first demonstration would pass for the wrong reason. What a test -/// needs to see of the traffic is recorded inside the browser instead, by -/// the recorder in , which reads the request bodies and -/// the responses the page actually sent and received. -/// -/// -/// It answers on a plain socket and pays no attention to the host name in -/// the request, rather than using the framework's own listener, because -/// that one routes by the name and registering any name but localhost -/// needs an administrator on Windows. A developer would then get a -/// different run from CI, which is the thing most likely to let a fault -/// through. Here every publisher site name reaches the same server, and -/// the browsers are told to resolve those names to this machine, so the -/// only thing separating one site from another is the name in the -/// request, which is exactly what a browser uses to decide what is third -/// party. -/// -/// -public sealed class PageServer : IDisposable -{ - private readonly TcpListener _listener; - private readonly CancellationTokenSource _stopping = new(); - private readonly ConcurrentDictionary _pages = new( - StringComparer.OrdinalIgnoreCase); - - /// - /// Method, host and path of every request served, newest last. A test - /// reads it while the server is live, so it is a queue that takes a - /// snapshot when enumerated rather than one that throws. - /// - public ConcurrentQueue RequestLog { get; } = new(); - - /// The port the sites are served on. - public int Port { get; } - - public PageServer() - { - _listener = new TcpListener(IPAddress.Loopback, 0); - _listener.Start(); - Port = ((IPEndPoint)_listener.LocalEndpoint).Port; - _ = Task.Run(Accept); - } - - /// - /// Puts a page at a path. The same path may be replaced between - /// navigations, which is how a test changes what the second page view - /// carries. - /// - public void Put(string path, string html) - => _pages[Normalise(path)] = html; - - /// - /// The address of a page on one of the publisher sites. The cache - /// buster is on every navigation, because a page held in the browser's - /// cache would run the recorder from an earlier test. - /// - public string UrlFor(string site, string path) - => $"http://{site}:{Port}{Normalise(path)}" - + $"?v={DateTime.UtcNow.Ticks}"; - - /// Every request served since the last clear. - public IReadOnlyList Requests => new List(RequestLog); - - public void ClearRequests() => RequestLog.Clear(); - - private async Task Accept() - { - while (_stopping.IsCancellationRequested == false) - { - TcpClient client; - try - { - client = await _listener.AcceptTcpClientAsync( - _stopping.Token).ConfigureAwait(false); - } - catch (Exception) - { - // The listener was stopped, which is how a test ends. - return; - } - _ = Task.Run(() => Serve(client)); - } - } - - private async Task Serve(TcpClient client) - { - using (client) - { - try - { - using var stream = client.GetStream(); - using var reader = new StreamReader( - stream, Encoding.ASCII, false, 1024, leaveOpen: true); - var requestLine = await reader.ReadLineAsync() - .ConfigureAwait(false); - if (string.IsNullOrEmpty(requestLine)) - { - return; - } - var host = string.Empty; - string? header; - // Headers are read to the blank line so the browser's - // request is fully consumed, and the host is kept because - // the log is the one place a test can see which site a - // request was for. - while (string.IsNullOrEmpty( - header = await reader.ReadLineAsync().ConfigureAwait(false)) - == false) - { - if (header!.StartsWith( - "Host:", StringComparison.OrdinalIgnoreCase)) - { - host = header.Substring(5).Trim(); - } - } - var parts = requestLine.Split(' '); - var method = parts.Length > 0 ? parts[0] : "GET"; - var path = Normalise(parts.Length > 1 ? parts[1] : "/"); - RequestLog.Enqueue($"{method} {host}{path}"); - - var found = _pages.TryGetValue(path, out var html); - var body = Encoding.UTF8.GetBytes(found ? html! : "not here"); - var head = new StringBuilder() - .Append(found ? "HTTP/1.1 200 OK\r\n" - : "HTTP/1.1 404 Not Found\r\n") - .Append("Content-Type: text/html; charset=utf-8\r\n") - .Append("Content-Length: ") - .Append(body.Length) - .Append("\r\n") - // Nothing here may be reused between page views, or a - // test that navigates twice measures the first page - // view twice. - .Append("Cache-Control: no-store, no-cache, must-revalidate\r\n") - .Append("Connection: close\r\n\r\n") - .ToString(); - var headBytes = Encoding.ASCII.GetBytes(head); - await stream.WriteAsync(headBytes).ConfigureAwait(false); - await stream.WriteAsync(body).ConfigureAwait(false); - await stream.FlushAsync().ConfigureAwait(false); - } - catch (Exception) - { - // A browser that went away mid response is not a failure of - // the thing under test, and the assertions are on what the - // page recorded rather than on what this served. - } - } - } - - private static string Normalise(string path) - { - var trimmed = path.Split('?')[0].Split('#')[0]; - return trimmed.StartsWith("/", StringComparison.Ordinal) - ? trimmed - : "/" + trimmed; - } - - public void Dispose() - { - _stopping.Cancel(); - try - { - _listener.Stop(); - } - catch (Exception) - { - // Already stopped. - } - _stopping.Dispose(); - } -} diff --git a/Browser51Did/Pages.cs b/Browser51Did/Pages.cs deleted file mode 100644 index 5495cc6..0000000 --- a/Browser51Did/Pages.cs +++ /dev/null @@ -1,324 +0,0 @@ -#nullable enable - -using System; -using System.Collections.Generic; -using System.Text; - -namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; - -/// -/// The real pages a browser loads in these tests, being a publisher's page -/// with the preference platform and the client script on it in one -/// arrangement or another. -/// -/// Every page starts with the recorder, which is the only thing on any of -/// them that is not a publisher's own markup. It wraps the two ways a page -/// makes a request and the console, so a test can read what the page -/// actually sent, what came back and what was logged, without a proxy in -/// front of the cloud. A proxy would put the cloud on the page's own -/// origin, and the whole first demonstration is about the cloud being a -/// third party to the page. -/// -/// -/// Nothing on these pages is publisher code in the sense the design means -/// it. The recorder is the test's instrument, and where a page carries a -/// handler, the test that uses that page says why. -/// -/// -public static class Pages -{ - /// - /// The object name a page uses when it does not ask for another one, - /// which is what the client script and the preference platform both - /// fall back to. - /// - public const string DefaultObjectName = "fod"; - - /// - /// A publisher's static vendor consent string, as the platform's own - /// demo page carries one. The platform only rewrites the purpose - /// consent bits and the dates in it. - /// - public const string PublisherVendorString = - "CPYBSvoPYBSvoO3AAAENAwCAAAAAAAAAAAAAAAAAAAAA"; - - /// - /// The recorder. It is first on every page, before anything is loaded - /// from the cloud, so nothing the page does afterwards escapes it. - /// - public const string Recorder = @" -"; - - /// - /// A handler registered on the client script's object as soon as it - /// exists, so a test can assert that a change of answer reached page - /// code. It waits for the object rather than assuming the script has - /// finished, because the script tag is asynchronous. - /// - /// This is the one piece of publisher code any of these pages carries, - /// and only the two tests about a change of answer use it. - /// - /// - public static string ChangeWatcher(string objectName) => $@" -"; - - /// - /// The client script's own tag, exactly as a publisher writes it. - /// - public static string ClientScriptTag(string? objectName = null) - => $""; - - /// - /// Settings on the preference platform's tag. Every one of them is an - /// attribute the publisher writes, and the defaults here are the ones - /// the platform's own demo page uses. - /// - public sealed class PlatformSettings - { - /// The network a choice is shared across. Null turns - /// sharing off, because the visitor cannot be asked to share with - /// a group nobody has named. - public string? NetworkName { get; set; } = Harness.NetworkName; - - /// Whether the standard answer is offered as well as the - /// personalized one. - public bool ShowStandard { get; set; } = true; - - /// The name the platform looks for the client script's - /// object under. Null leaves the attribute off, which is the - /// documented way of saying the default. - public string? ObjectName { get; set; } - - /// How long the platform waits for the third party cookie - /// answer before carrying on without one. - public int TimeoutMs { get; set; } = 4500; - } - - /// - /// The preference platform's tag. The action and the alternative are - /// page hooks rather than a second script, so a test can see them fire - /// and so that nothing here loads a second copy of the client script, - /// which two of the tests assert did not happen. - /// - public static string PlatformTag(PlatformSettings settings) - { - var attributes = new List - { - $"data-resource-key=\"{Harness.Resource}\"", - "data-action-url=\"javascript:window.__51dTest.actions" - + ".push('{preference}')\"", - $"data-tcf-vendor=\"{PublisherVendorString}\"", - "data-brand-name=\"Fifty One Times\"", - "data-brand-terms-url=\"https://example.com/privacy\"", - "data-alt-name=\"Subscribe\"", - "data-alt-url=\"javascript:window.__51dTest.altFired = true\"", - $"data-show-standard=\"{(settings.ShowStandard ? "true" : "false")}\"", - $"data-timeout=\"{settings.TimeoutMs}\"", - }; - if (settings.NetworkName is null) - { - attributes.Add("data-use-third-party-cookies=\"false\""); - } - else - { - attributes.Add($"data-network-name=\"{settings.NetworkName}\""); - } - if (settings.ObjectName is not null) - { - attributes.Add($"data-object-name=\"{settings.ObjectName}\""); - } - return $""; - } - - /// - /// A consent platform, being the stub the framework's own - /// specification tells every publisher to put in the page, and a fake - /// platform behind it that answers only what the specification - /// requires, which is ping, addEventListener and a callback carrying - /// tcString and eventStatus. - /// - /// It does not deliver anything until the test asks it to with - /// window.__51dCmp.deliver(), so the test decides the ordering - /// rather than a timer. - /// - /// - public static string ConsentPlatform(string tcString) => $@" -"; - - /// Wraps the parts into a page. - public static string Page(string title, params string[] parts) - { - var body = new StringBuilder(); - body.Append("\n\n\n") - .Append("\n") - .Append("").Append(title).Append("\n") - .Append(Recorder) - .Append("\n\n\n

") - .Append(title) - .Append("

\n"); - foreach (var part in parts) - { - body.Append(part).Append('\n'); - } - return body.Append("\n\n").ToString(); - } -} diff --git a/Browser51Did/PlatformAcceptanceTests.cs b/Browser51Did/PlatformAcceptanceTests.cs index 58fe854..ebcd3ab 100644 --- a/Browser51Did/PlatformAcceptanceTests.cs +++ b/Browser51Did/PlatformAcceptanceTests.cs @@ -28,7 +28,7 @@ public class PlatformAcceptanceTests : Browser51DidTestBase /// The second card is up before any of that finished, because nothing /// waits between the two cards. /// - [TestMethod] + [Browser51DidTest] public void CommonPath_PlatformThenScript_Chrome() => CommonPath(Chrome, platformFirst: true); @@ -37,7 +37,7 @@ public void CommonPath_PlatformThenScript_Chrome() /// because anything the shared choice travels on behaves differently /// between them. /// - [TestMethod] + [Browser51DidTest] public void CommonPath_PlatformThenScript_Firefox() => CommonPath(Firefox, platformFirst: true); @@ -48,24 +48,17 @@ public void CommonPath_PlatformThenScript_Firefox() /// design that only worked one way round would pass the test above and /// fail on half the customers' pages. /// - [TestMethod] + [Browser51DidTest] public void CommonPath_ScriptThenPlatform_Chrome() => CommonPath(Chrome, platformFirst: false); private void CommonPath(string browser, bool platformFirst) { - using var server = new PageServer(); RequireMarketingIdentifiers(); - var settings = new Pages.PlatformSettings(); - var platform = Pages.PlatformTag(settings); - var script = Pages.ClientScriptTag(); - server.Put("/", Pages.Page( - "Common path", - platformFirst ? platform : script, - platformFirst ? script : platform)); - - using var visitor = NewVisitor(browser, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(browser); + visitor.Go( + Harness.SiteA, + platformFirst ? Routes.Common : Routes.CommonScriptFirst); // The first round. Nobody has been asked yet, so nothing may be // created, which is the rule the whole programme exists for. @@ -90,39 +83,35 @@ private void CommonPath(string browser, bool platformFirst) // The visitor answers. visitor.WaitForPlatform(); visitor.WaitForCard("preferences"); + // Records when each round finishes and when the share card arrives, + // for the failure message below. It changes nothing on the page. + visitor.StartTimeline(); var roundsBefore = visitor.ClientRequests().Count; - // How many rounds had already finished, which is what the share - // card must appear before any more of. The page may well have - // finished more than one by now, because the snippets it was asked - // to run produce a round of their own, so this is read rather than - // assumed to be one. - var doneBefore = visitor.ClientRequests().Count(r => r.Done); visitor.Press("standard"); - // The second card is up before the refresh has finished. Both - // readings come from one call, so there is no gap between them in - // which the answer could change. - (bool CardVisible, int RoundsDone) atShare = (false, -1); + // The second card is up before the refresh has finished. How many + // rounds had finished is counted inside the page, once at the moment + // of the press and once at the first frame the card is visible, so + // no round can finish unseen in the time it takes to ask the page + // from here. The page may well have finished more than one round + // before the press, because the snippets it was asked to run produce + // a round of their own, so the count at the press is read rather + // than assumed to be one. Harness.Until( - () => - { - var seen = visitor.Observe("share"); - if (seen.CardVisible && atShare.RoundsDone < 0) - { - atShare = seen; - } - return atShare.RoundsDone >= 0; - }, + () => visitor.RoundsFinishedAround().AtShareVisible >= 0, $"the share card appeared in {browser}. The console said: " - + $"{string.Join(" | ", visitor.Console())}"); + + $"{string.Join(" | ", visitor.Console())}. What the page " + + $"recorded, frame by frame, was: {visitor.Timeline()}"); + var (doneAtPress, doneAtShare) = visitor.RoundsFinishedAround(); Assert.AreEqual( - doneBefore, - atShare.RoundsDone, + doneAtPress, + doneAtShare, "the share card must follow the first card at once, with " - + "nothing waiting on the refresh. When it appeared the client " - + $"script had finished {atShare.RoundsDone} rounds rather than " - + $"the {doneBefore} it had finished at the click, so something " - + "waited."); + + "nothing waiting on the refresh. When it first became visible " + + $"the client script had finished {doneAtShare} rounds rather " + + $"than the {doneAtPress} it had finished at the press, so " + + "something waited. What the page recorded, frame by frame, " + + $"was: {visitor.Timeline()}"); // Exactly one further request, carrying the answer once and every // snippet result the page had worked out. @@ -204,7 +193,7 @@ private void CommonPath(string browser, bool platformFirst) /// identifier from the answer with the signal source recorded as /// direct. /// - [TestMethod] + [Browser51DidTest] public void SharedChoice_SecondSite_NoDialogAndDirectFlag_Chrome() => SharedChoice(Chrome); @@ -213,25 +202,19 @@ public void SharedChoice_SecondSite_NoDialogAndDirectFlag_Chrome() /// third party cookie and that is the thing the two browsers treat /// differently. /// - [TestMethod] + [Browser51DidTest] public void SharedChoice_SecondSite_NoDialogAndDirectFlag_Firefox() => SharedChoice(Firefox); private void SharedChoice(string browser) { - using var server = new PageServer(); + Harness.RequireSecureCookies(); RequireMarketingIdentifiers(); RequireSharing(); - var page = Pages.Page( - "Shared choice", - Pages.PlatformTag(new Pages.PlatformSettings()), - Pages.ClientScriptTag()); - server.Put("/", page); - - using var visitor = NewVisitor(browser, server); + using var visitor = NewVisitor(browser); // Site A, where the visitor answers and agrees to share. - visitor.Go(Harness.SiteA, "/"); + visitor.Go(Harness.SiteA, Routes.Common); visitor.WaitForCard("preferences"); visitor.Press("standard"); visitor.WaitForCard("share"); @@ -243,7 +226,7 @@ private void SharedChoice(string browser) + $"console said: {string.Join(" | ", visitor.Console())}"); // Site B, a first visit, in the same browser. - visitor.Go(Harness.SiteB, "/"); + visitor.Go(Harness.SiteB, Routes.Common); visitor.WaitForPlatform(); visitor.WaitForClientRounds(1); @@ -312,19 +295,12 @@ private void SharedChoice(string browser) /// different identifier comes back, and page code that registered a /// change handler is told. /// - [TestMethod] + [Browser51DidTest] public void ChangeOfAnswer_SamePage_NewIdentifierAndOnChange() { - using var server = new PageServer(); RequireMarketingIdentifiers(); - server.Put("/", Pages.Page( - "Change of answer", - Pages.PlatformTag(new Pages.PlatformSettings()), - Pages.ClientScriptTag(), - Pages.ChangeWatcher(Pages.DefaultObjectName))); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.Change); visitor.WaitForCard("preferences"); visitor.Press("standard"); Harness.Until( @@ -402,21 +378,12 @@ public void ChangeOfAnswer_SamePage_NewIdentifierAndOnChange() /// here would be asserting that the cache does not work. /// /// - [TestMethod] + [Browser51DidTest] public void ChangeOfAnswer_AcrossPages_TheSecondPageCarriesTheNewAnswer() { - using var server = new PageServer(); RequireMarketingIdentifiers(); - var page = Pages.Page( - "Two pages", - Pages.PlatformTag(new Pages.PlatformSettings()), - Pages.ClientScriptTag(), - Pages.ChangeWatcher(Pages.DefaultObjectName)); - server.Put("/one", page); - server.Put("/two", page); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/one"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.TwoOne); visitor.WaitForCard("preferences"); visitor.Press("standard"); Harness.Until( @@ -444,7 +411,7 @@ public void ChangeOfAnswer_AcrossPages_TheSecondPageCarriesTheNewAnswer() "the change was to personalized."); // The second page, in the same tab. - visitor.Go(Harness.SiteA, "/two"); + visitor.Go(Harness.SiteA, Routes.TwoTwo); Harness.Until( () => visitor.Identifier() != "", "the second page settled on an identifier. The console said: " @@ -477,17 +444,11 @@ public void ChangeOfAnswer_AcrossPages_TheSecondPageCarriesTheNewAnswer() /// any other, so it creates an identifier with the direct flag, and /// there is nothing to share, so no second card is offered. /// - [TestMethod] + [Browser51DidTest] public void AlternativeAnswer_CreatesNonMarketingAndFiresTheAction() { - using var server = new PageServer(); - server.Put("/", Pages.Page( - "Alternative", - Pages.PlatformTag(new Pages.PlatformSettings()), - Pages.ClientScriptTag())); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.Common); visitor.WaitForCard("preferences"); visitor.Press("alternative"); diff --git a/Browser51Did/PlatformAddsClientScriptTests.cs b/Browser51Did/PlatformAddsClientScriptTests.cs index 51fb5d8..edf9dc8 100644 --- a/Browser51Did/PlatformAddsClientScriptTests.cs +++ b/Browser51Did/PlatformAddsClientScriptTests.cs @@ -28,17 +28,12 @@ public class PlatformAddsClientScriptTests : Browser51DidTestBase /// The whole of it in one page view, from the script arriving to the /// identifier coming back. /// - [TestMethod] + [Browser51DidTest] public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() { - using var server = new PageServer(); RequireMarketingIdentifiers(); - server.Put("/", Pages.Page( - "Platform with no client script", - Pages.PlatformTag(new Pages.PlatformSettings()))); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.PlatformOnly); visitor.WaitForPlatform(); // The script the platform added, named by where it came from and @@ -49,7 +44,7 @@ public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() + string.Join(" | ", visitor.Console())); var added = AddedClientScript(visitor)!; Assert.IsTrue( - added.StartsWith(Harness.BaseUrl!, StringComparison.OrdinalIgnoreCase), + added.StartsWith(Harness.CloudUrl!, StringComparison.OrdinalIgnoreCase), "the script must come from the cloud that served the platform, " + "because that is the only cloud the platform knows about. It " + $"came from {added}."); @@ -143,20 +138,12 @@ public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() /// and the platform then asks for the script under that name, finds it /// under that name, and says which name it used. /// - [TestMethod] + [Browser51DidTest] public void ObjectNameAttribute_NamesTheObjectEverywhere() { - const string objectName = "fiftyOneData"; - using var server = new PageServer(); - server.Put("/", Pages.Page( - "Platform with a named object", - Pages.PlatformTag(new Pages.PlatformSettings - { - ObjectName = objectName, - }))); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + const string objectName = Harness.NamedObject; + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.NamedObject); visitor.WaitForPlatform(); Harness.Until( @@ -175,7 +162,7 @@ public void ObjectNameAttribute_NamesTheObjectEverywhere() visitor.HasTheNewClientScript(objectName), $"the object under '{objectName}' must be the client script's."); Assert.IsFalse( - visitor.HasClientObject(Pages.DefaultObjectName), + visitor.HasClientObject(Harness.DefaultObjectName), "nothing may be put under the default name when the publisher " + "asked for another one, because two objects would be two " + "instances and the page would warn about the second."); @@ -193,20 +180,15 @@ public void ObjectNameAttribute_NamesTheObjectEverywhere() /// The attribute left off uses the default name, which is what the /// documentation says and what an existing page relies on. /// - [TestMethod] + [Browser51DidTest] public void ObjectNameAttributeAbsent_UsesTheDefaultName() { - using var server = new PageServer(); - server.Put("/", Pages.Page( - "Platform with no object name", - Pages.PlatformTag(new Pages.PlatformSettings()))); - - using var visitor = NewVisitor(Chrome, server); - visitor.Go(Harness.SiteA, "/"); + using var visitor = NewVisitor(Chrome); + visitor.Go(Harness.SiteA, Routes.PlatformOnly); visitor.WaitForPlatform(); - visitor.WaitForClientObject(Pages.DefaultObjectName); + visitor.WaitForClientObject(Harness.DefaultObjectName); Assert.IsTrue( - visitor.HasTheNewClientScript(Pages.DefaultObjectName), + visitor.HasTheNewClientScript(Harness.DefaultObjectName), "with no name asked for, the object is the default one."); } diff --git a/Browser51Did/README.md b/Browser51Did/README.md new file mode 100644 index 0000000..e9986b1 --- /dev/null +++ b/Browser51Did/README.md @@ -0,0 +1,119 @@ +# 51Did browser acceptance tests + +Category `Browser51Did`. Fifteen tests proving, in real Chrome and Firefox, +that a 51Did is created only after every other piece of data is complete and +only from an answer the visitor actually gave, with the 51Degrees Preference +Management Platform (PMP) and the client script on a publisher's page. + +They drive a **demo**, a small web app that serves every page the tests +load. The dotnet demo is `Examples/Cloud/PreferenceManagement-Web` in +[device-detection-dotnet-examples](https://github.com/51Degrees/device-detection-dotnet-examples). +The pages, the recorder that watches them, the change watcher and the stub +consent platform are plain static files in that demo, so a demo in another +language copies them, fills the same placeholders, and these same tests then +prove it behaves the same. + +## What they prove + +1. A choice made on one site is read on another, the visitor is not asked + again, and a 51Did is still created with the signal source recorded as + direct. Chrome and Firefox, because the choice travels on a third party + cookie. +2. A page with a consent platform and no PMP sends the framework string, + the cloud decodes it, and the 51Did records that the usage was decoded. +3. The common path. The first request carries no answer and creates + nothing, the visitor answers, the second card follows at once, exactly + one further request carries the answer and every snippet result, and the + 51Did comes back. Both tag orders. +4. A change of answer on one page makes a new 51Did with the sequence one + higher and tells a change handler. +5. A change of answer carried to the next page in the same tab. +6. The alternative answer creates a `non-marketing` 51Did with the direct + flag, fires the publisher's action, and offers no second card. +7. Whether the regulation applies, read from the client script. This cannot + pass until the IsGdpr property is released in the cloud, and reports + inconclusive with that reason until then. +8. A page with nowhere to get an answer from creates nothing, says so once, + and stores a record carrying no answer. +9. A page carrying the PMP and no client script tag gets the script added by + the PMP from the cloud that served it. +10. `data-object-name` names the object everywhere, and its absence gives + the default `fod`. + +Every assertion is on an ordering of requests and page states, never on a +clock. The guard, `Harness.RequireTheNewClientScript`, fails every test in a +class unless the client script the pages load carries the new template's +user prompt block. + +## The pages + +| Route | Markup | Tests | +| --- | --- | --- | +| `common` | PMP, then client script | common path, shared choice (as `site-a.localtest` then `site-b.localtest`), alternative, both IsGdpr | +| `common-script-first` | client script, then PMP | common path, other tag order | +| `change` | PMP, client script, change watcher | change of answer on one page | +| `two/one`, `two/two` | the same as `change` | change of answer across pages | +| `consent` | stub consent platform, then client script | consent platform only | +| `no-platform` | client script alone | no platform at all | +| `platform-only` | PMP alone | PMP adds the client script, default object name | +| `named-object` | PMP with `data-object-name="fiftyOneData"` | object name attribute | + +Each route is served under a mode prefix. `/cloud/` pages load the client +script and the PMP straight from the cloud, and the client script posts to +the cloud's `/api/v4/json`. `/pipeline/` pages have the demo's own pipeline +serve the client script, which posts to the path the demo's descriptor names +in `ExampleDescriptor.JsonEndpointPath`. The tests read that path from the +mode rather than assuming one. + +## Running them + +The demo is started once for the run through `Examples/ExampleApps.cs`, as +the Contract tests start an example. + +| Variable | Meaning | +| --- | --- | +| `51DEGREES_CLOUD_ENDPOINT` | The cloud, including its `api/v4` path. Handed to the demo unchanged. | +| `51DEGREES_RESOURCE_KEY` | A resource key the cloud creates 51Dids for standard and personalized answers with. Handed to the demo unchanged, and never printed, because every message a test produces has it taken out. | +| `DEMO_LANG` | The demo to launch from the sibling checkout. `dotnet` unless set. | +| `DEMO_URL` | A demo already running, which is used instead of launching one. | +| `DEMO_MODE` | `cloud` unless set to `pipeline`. | +| `CLOUD_ROOT_URL` | Not used by these tests, but the suite's assembly set up requires it. | + +```bash +env CLOUD_ROOT_URL="http://localhost:5050/" \ + 51DEGREES_CLOUD_ENDPOINT="http://localhost:5050/api/v4/" \ + 51DEGREES_RESOURCE_KEY="" \ + dotnet test --filter TestCategory=Browser51Did +``` + +`env` is used because bash cannot export a variable whose name starts with a +digit. In PowerShell write `${env:51DEGREES_RESOURCE_KEY} = '...'`. + +With the two input variables unset every test reports inconclusive with the +reason. A resource key that cannot create reports inconclusive with the +cloud's own refusal, and so does a cloud that will not hold a shared choice. +The shared choice needs the browser to keep the cloud's `Secure` +`SameSite=None` cookie, which both browsers did from plain HTTP on +`localhost` when measured, so it skips only for a plain HTTP cloud reached by +another name. + +## Adding a demo in another language + +1. Copy the demo's static files and templates, and fill the same + placeholders from the same two variables. +2. Serve every route under `/cloud/`, answering any host name, because the + tests load the pages as `site-a.localtest` and `site-b.localtest` on the + demo's port. +3. Add an entry to `ExampleApps.Demos` saying how to launch it. +4. Run the category with `DEMO_LANG` set to that entry. + +## What only the cloud mode proves today + +The `/pipeline/` mode needs the demo's own pipeline to render the user +prompt block, which needs a 51Did element in that pipeline and a JavaScript +builder carrying the new template. Until a demo serves those pages every +run uses `/cloud/`. Two things will need attention when it does. The PMP +recognises a client script tag only by the cloud's path, `/api/v4/.js`, +so on a pipeline page it may add the cloud's script beside the demo's own, +and the tests about the PMP adding a script prove the cloud's script in +either mode. diff --git a/Browser51Did/TcString.cs b/Browser51Did/TcString.cs index 11ea99f..44ea808 100644 --- a/Browser51Did/TcString.cs +++ b/Browser51Did/TcString.cs @@ -11,13 +11,18 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// are set, because those are the only ones the server reads when it works /// out a usage from a string. /// -/// This is the same construction as TcStringBuilder in -/// Did/Tests/FiftyOne.Did.OnPremise.Tests. It is repeated here -/// rather than referenced because this project is deliberately outside the -/// solution and takes no project references, and because a browser test -/// that quietly changed when a unit test helper changed would be worse +/// This is the same construction as TcStringBuilder in the cloud +/// repository's Did/Tests/FiftyOne.Did.OnPremise.Tests. It is +/// repeated here because that is another repository, and because a browser +/// test that quietly changed when a unit test helper changed would be worse /// than a small repetition. /// +/// +/// Every language's demo delivers from its stub +/// consent platform, which is +/// AAAAAAAAAAAAAAAAAAAAAAAAAP_wAAAA, and the consent test checks the +/// string that reached the cloud is exactly this. +/// /// public static class TcString { diff --git a/Browser51Did/Visitor.cs b/Browser51Did/Visitor.cs index 4f8c1ee..ffe696a 100644 --- a/Browser51Did/Visitor.cs +++ b/Browser51Did/Visitor.cs @@ -60,10 +60,12 @@ public override string ToString() } /// -/// One visitor, being a browser and the publisher's website it is looking -/// at. Everything a test asserts on is read back through here, so an -/// assertion reads as a sentence about the page rather than as a lump of -/// JavaScript. +/// One visitor, being a browser looking at the demo's pages as the +/// publisher's sites. Everything a test asserts on is read back through +/// here, so an assertion reads as a sentence about the page rather than as a +/// lump of JavaScript. What it reads was put on the page by the demo, so a +/// demo in another language that serves the same pages passes the same +/// reads. /// public sealed class Visitor : IDisposable { @@ -105,25 +107,25 @@ function visible(el) { public IWebDriver Driver { get; } - public PageServer Server { get; } - /// The browser's name, for a failure message. public string BrowserName { get; } - public Visitor(IWebDriver driver, PageServer server, string browserName) + public Visitor(IWebDriver driver, string browserName) { Driver = driver; - Server = server; BrowserName = browserName; } - /// Loads a page and waits for the document to settle. - public void Go(string site, string path) + /// + /// Loads one of the demo's pages as one of the publisher sites, and + /// waits for the document to settle. + /// + public void Go(string site, string route) { - Driver.Navigate().GoToUrl(Server.UrlFor(site, path)); + Driver.Navigate().GoToUrl(Demo.Chosen.PageUrl(site, route)); Harness.Until( () => Script("return document.readyState;") == "complete", - $"the page at {site}{path} finished loading"); + $"the {route} page at {site} finished loading"); } public T? Script(string body, params object[] arguments) @@ -156,12 +158,14 @@ public IReadOnlyList Requests() /// /// The client script's own requests, being the ones to the endpoint it - /// posts its evidence to. + /// posts its evidence to, which the demo's mode names rather than this + /// assuming the cloud's. /// public IReadOnlyList ClientRequests() => Requests() .Where(r => r.Url.Contains( - "/api/v4/json", StringComparison.OrdinalIgnoreCase)) + Demo.Chosen.JsonEndpointPath, + StringComparison.OrdinalIgnoreCase)) .ToList(); /// @@ -243,7 +247,7 @@ public IReadOnlyList LocalStorageKeys() #region The client script's object /// Whether the client script's object exists on the page. - public bool HasClientObject(string objectName = Pages.DefaultObjectName) + public bool HasClientObject(string objectName = Harness.DefaultObjectName) => Script( $"return typeof window['{objectName}'] === 'object'" + $" && window['{objectName}'] !== null;"); @@ -254,7 +258,7 @@ public bool HasClientObject(string objectName = Pages.DefaultObjectName) /// answer has reached the cloud, so an empty string is the normal /// state of a first request that carried no answer. /// - public string Identifier(string objectName = Pages.DefaultObjectName) + public string Identifier(string objectName = Harness.DefaultObjectName) => Script( $@"var o = window['{objectName}']; return (o && o.fodid && o.fodid.idprobglobal) || '';") ?? ""; @@ -266,7 +270,7 @@ public string Identifier(string objectName = Pages.DefaultObjectName) /// on what the endpoint served the test host. /// public bool HasTheNewClientScript( - string objectName = Pages.DefaultObjectName) + string objectName = Harness.DefaultObjectName) => Script( $"var o = window['{objectName}'];" + " return !!(o && typeof o.refresh === 'function');"); @@ -276,7 +280,7 @@ public bool HasTheNewClientScript( /// page sees it. Empty where the property has no value. /// public string ThirdPartyCookies( - string objectName = Pages.DefaultObjectName) + string objectName = Harness.DefaultObjectName) => Script( $@"var o = window['{objectName}']; var d = o && o.device; @@ -291,7 +295,7 @@ public string ThirdPartyCookies( /// properties the resource key happens to carry. /// public IReadOnlyList SnippetValueNames( - string objectName = Pages.DefaultObjectName) + string objectName = Harness.DefaultObjectName) => JsonSerializer.Deserialize>( Read($@" var prefix = '{objectName}_data_'; @@ -319,7 +323,7 @@ public IReadOnlyList SnippetValueNames( /// silently checks nothing. /// /// - public string? CacheRecord(string objectName = Pages.DefaultObjectName) + public string? CacheRecord(string objectName = Harness.DefaultObjectName) { var value = Script($@" try {{ @@ -335,40 +339,13 @@ public IReadOnlyList SnippetValueNames( return string.IsNullOrEmpty(value) ? null : value; } - /// - /// Whether a card is showing and how many rounds the client script has - /// finished, read together in one call, so a test can say that one was - /// true while the other had not moved. Two separate reads would leave a - /// gap in which the answer could change, and the ordering is the whole - /// point of the assertion. - /// - public (bool CardVisible, int RoundsDone) Observe(string card) - { - var answer = Script($@" - var root = pmpRoot(); - var el = root - ? root.querySelector('[data-card=""{card}""]') : null; - var showing = visible(el); - var done = 0; - var all = (window.__51dTest && window.__51dTest.requests) || []; - for (var i = 0; i < all.length; i++) {{ - if (all[i].done - && all[i].url.indexOf('/api/v4/json') !== -1) {{ - done++; - }} - }} - return (showing ? '1' : '0') + '|' + done;") ?? "0|0"; - var parts = answer.Split('|'); - return (parts[0] == "1", int.Parse(parts[1], CultureInfo.InvariantCulture)); - } - /// /// Whether the visitor is somewhere the regulation applies, as the /// client script resolved it. Empty where the property has no value, /// which is what a resource key that does not carry it produces and /// what every key produces until the change adding it is released. /// - public string IsGdpr(string objectName = Pages.DefaultObjectName) + public string IsGdpr(string objectName = Harness.DefaultObjectName) => Script( $@"var o = window['{objectName}']; var groups = ['derived', 'device']; @@ -477,6 +454,12 @@ public void Press(string action) if (!root) {{ return false; }} var el = root.querySelector('[data-action=""{action}""]'); if (!el) {{ return false; }} + if (window.__51dTimeline) {{ + window.__51dTimeline.atPress = + window.__51dTimeline.finished(); + window.__51dTimeline.push( + performance.now(), 'pressed {action}'); + }} el.click(); return true;"); Assert.IsTrue( @@ -537,14 +520,137 @@ public void WaitForClientRounds(int rounds) + $"The console said: {string.Join(" | ", Console())}"); /// - /// What the browser asked the publisher's website for, which is the - /// one thing that separates a page that never loaded from a page that - /// loaded and then did nothing. + /// Which page the browser is on and how far it got, which is what + /// separates a page that never loaded from a page that loaded and then + /// did nothing. /// public string ServedSoFar() - => Server.Requests.Count == 0 - ? "the browser asked this website for nothing at all" - : "the website served " + string.Join(", ", Server.Requests); + { + try + { + return "the browser is on " + + Script( + "return location.href + ' titled \"' + document.title" + + " + '\" in state ' + document.readyState;"); + } + catch (WebDriverException error) + { + return "the browser could not say which page it is on " + + $"({error.Message})"; + } + } + + #region Timeline + + /// + /// Starts recording, frame by frame inside the page, when each of the + /// client script's requests is first seen and when it finishes, when + /// the share card is put into the dialog and when it first becomes + /// visible, with adding the moment of each press. + /// + /// The ordering assertion about the share card reads its two counts + /// from here, through . It used to + /// poll the page from the test, and failed 3 times in 7 runs before + /// these tests moved. On 15 September 2026 this timeline caught one of + /// those failures, with the card visible at 302 ms and the answered + /// round finishing at 500 ms, so the card had not waited. The poll had + /// first seen the card after 500 ms, and counted the round that had + /// finished in between. Counting inside the page, at the press and at + /// the first frame the card is visible, measures the same thing + /// without the delay of reaching into the page from outside. + /// + /// + public void StartTimeline() + => Script(@" + if (window.__51dTimeline) { return null; } + var t0 = performance.now(); + var events = []; + var path = arguments[0].toLowerCase(); + function finished() { + var n = 0; + var all = (window.__51dTest && window.__51dTest.requests) || []; + for (var i = 0; i < all.length; i++) { + if (all[i].done + && all[i].url.toLowerCase().indexOf(path) !== -1) { + n++; + } + } + return n; + } + var tl = window.__51dTimeline = { + push: function (at, what) { + events.push(Math.round(at - t0) + 'ms ' + what); + }, + read: function () { return events.join(' | '); }, + finished: finished, + atPress: -1, + atShareVisible: -1 + }; + var seen = {}; + var cardIn = false; + var cardVisible = false; + function tick() { + var now = performance.now(); + var all = (window.__51dTest && window.__51dTest.requests) || []; + for (var i = 0; i < all.length; i++) { + var r = all[i]; + if (r.url.toLowerCase().indexOf(path) === -1) { continue; } + if (!seen['s' + i]) { + seen['s' + i] = true; + tl.push(now, 'request ' + i + ' seen' + + (r.done ? ' already finished' : '') + + (r.body.indexOf('id.usage=') !== -1 + ? ' carrying an answer' : '')); + } + if (r.done && !seen['d' + i]) { + seen['d' + i] = true; + tl.push(now, 'request ' + i + ' finished'); + } + } + var root = pmpRoot(); + var card = root + ? root.querySelector('[data-card=""share""]') : null; + if (card && !cardIn) { + cardIn = true; + tl.push(now, 'share card in the dialog'); + } + if (card && !cardVisible && visible(card)) { + cardVisible = true; + tl.atShareVisible = finished(); + tl.push(now, 'share card visible'); + } + if (events.length < 200) { requestAnimationFrame(tick); } + } + tl.push(t0, 'recording started'); + requestAnimationFrame(tick); + return null;", + Demo.Chosen.JsonEndpointPath); + + /// + /// How many of the client script's rounds had finished at the moment of + /// the last press, and at the first frame in which the share card was + /// visible, both counted inside the page, or -1 for a moment that has + /// not happened yet. + /// + public (int AtPress, int AtShareVisible) RoundsFinishedAround() + { + var answer = Script( + "var t = window.__51dTimeline;" + + " return t ? t.atPress + '|' + t.atShareVisible : '-1|-1';") + ?? "-1|-1"; + var parts = answer.Split('|'); + return ( + int.Parse(parts[0], CultureInfo.InvariantCulture), + int.Parse(parts[1], CultureInfo.InvariantCulture)); + } + + /// What the timeline recorded, oldest first. + public string Timeline() + => Script( + "return window.__51dTimeline ? window.__51dTimeline.read() : " + + "'no timeline was recorded';") ?? "unreadable"; + + #endregion /// Waits until the platform has put its dialog on the page. public void WaitForPlatform() @@ -564,7 +670,7 @@ public void WaitForCard(string card) /// Waits until the client script's object exists. public void WaitForClientObject( - string objectName = Pages.DefaultObjectName) + string objectName = Harness.DefaultObjectName) => Harness.Until( () => HasClientObject(objectName), $"an object called '{objectName}' appeared in {BrowserName}. " diff --git a/Examples/ExampleApps.cs b/Examples/ExampleApps.cs index 2b268af..ebd54d5 100644 --- a/Examples/ExampleApps.cs +++ b/Examples/ExampleApps.cs @@ -67,6 +67,85 @@ public static bool TryCreate(out IExampleApp app, out string skipReason) return true; } + private const string DemoLangVar = "DEMO_LANG"; + private const string DemoUrlVar = "DEMO_URL"; + + /// + /// The selected demo's language (DEMO_LANG, default "dotnet"). + /// + /// + /// A demo is chosen separately from the example the Contract tests + /// use, because the two are different apps. The GettingStarted-Web + /// examples prove a web integration works, and a demo carries every + /// page the Browser51Did tests drive, which each language mirrors so + /// that the same tests hold for all of them. DEMO_URL and DEMO_LANG + /// work the way EXAMPLE_URL and EXAMPLE_LANG do. + /// + public static string SelectedDemoLang => + Environment.GetEnvironmentVariable(DemoLangVar) ?? "dotnet"; + + /// + /// Attempts to create the demo for the current environment, being the + /// one already running at DEMO_URL, or else the DEMO_LANG demo + /// launched from its sibling checkout. Returns false when no demo is + /// registered for that language. + /// + public static bool TryCreateDemo( + out IExampleApp app, + out ExampleDescriptor descriptor, + out string skipReason) + { + Demos.TryGetValue(SelectedDemoLang, out descriptor); + var external = Environment.GetEnvironmentVariable(DemoUrlVar); + if (!string.IsNullOrEmpty(external)) + { + app = new ExternalExampleApp(new Uri(external)); + skipReason = null; + return true; + } + if (descriptor == null) + { + app = null; + skipReason = + $"No demo registered for DEMO_LANG='{SelectedDemoLang}'. " + + $"Known: {string.Join(", ", Demos.Keys)}."; + return false; + } + app = new SubprocessExampleApp(descriptor); + skipReason = null; + return true; + } + + /// + /// Per-language demos. Each serves the same pages under the same + /// routes, and reads its input data from the same two variables, + /// 51DEGREES_RESOURCE_KEY and 51DEGREES_CLOUD_ENDPOINT, so a demo in + /// another language is added here and nothing else in the suite + /// changes. + /// + public static readonly IReadOnlyDictionary Demos = + new Dictionary + { + ["dotnet"] = new ExampleDescriptor( + Lang: "dotnet", + WorkingDir: Path.Combine( + RepoPaths.SiblingsRoot, + "device-detection-dotnet-examples", + "Examples", "Cloud", "PreferenceManagement-Web"), + Command: "dotnet", + Args: new[] { "run", "-c", "Release", "--no-launch-profile" }, + ReadinessPath: "/cloud/common", + StartupTimeoutSeconds: 180, + BuildEnv: o => new Dictionary + { + ["51DEGREES_RESOURCE_KEY"] = o.ResourceKey, + // the endpoint includes the api/v4 path, as every + // other 51DEGREES_CLOUD_ENDPOINT reader expects + ["51DEGREES_CLOUD_ENDPOINT"] = new Uri(o.CloudEndpoint, "api/v4/").ToString(), + ["ASPNETCORE_URLS"] = $"http://localhost:{o.Port}", + }), + }; + /// Per-language launch descriptors. public static readonly IReadOnlyDictionary Descriptors = new Dictionary diff --git a/README.md b/README.md index d13262d..eab517a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ sibling directory and run it at their integration-test step. |---|---|---| | `Contract` | An example app serves `51Degrees.core.js`, client-side evidence flows back, and the server-rendered page shows a real detection result. | Cloud CI (per example, vs `:8080`) **and** every API CI (vs the public cloud). | | `CloudInternal` | Cloud response behaviour through a browser: cache reuse, COEP/CORP headers, third-party cookies, client-side overrides, and the per-browser JS endpoints. | Cloud CI only (vs `:8080`). | +| `Browser51Did` | The 51Did user prompt work in Chrome and Firefox, driving a demo's pages. See [Browser51Did/README.md](Browser51Did/README.md). | Cloud CI when something they prove changed (vs `:8080`, dotnet demo). | Select a subset with `--filter TestCategory=Contract` or `--filter TestCategory=CloudInternal`. @@ -42,6 +43,8 @@ and no keys are committed. | `ENTERPRISE_V4_LICENSE` | `CloudInternal` | License passed to the JS endpoint to unlock paid properties. | | `SELENIUM_URL` | optional | Selenium grid URL; omit for a local Chrome driver. | | `EXAMPLE_URL` / `EXAMPLE_LANG` | `Contract` | The example app to test (CI / local). | +| `51DEGREES_CLOUD_ENDPOINT` / `51DEGREES_RESOURCE_KEY` | `Browser51Did` | Handed unchanged to the demo, the same names every language's demo reads. | +| `DEMO_URL` / `DEMO_LANG` / `DEMO_MODE` | `Browser51Did` | The demo to test, and `cloud` or `pipeline` pages. | A missing variable only fails the test that reads it. diff --git a/TestsCommon/TestConfig.cs b/TestsCommon/TestConfig.cs index b80ae70..a231eeb 100644 --- a/TestsCommon/TestConfig.cs +++ b/TestsCommon/TestConfig.cs @@ -14,7 +14,15 @@ public class TestConfig public string PaidResourceKey => Require(_paidResourceKey); // Enterprise V4 license, passed to the JS endpoint to unlock paid properties. public string EnterpriseV4License => Require(_enterpriseV4License); + // Resource key a demo is started with. The same name is read by every + // language's demo, so the suite hands it over unchanged. + public string DemoResourceKey => Require(DemoResourceKeyVariable); + // Cloud endpoint a demo is started with, including the api/v4 path. + // Read by every language's demo under the same name. + public string DemoCloudEndpoint => Require(DemoCloudEndpointVariable); + public const string DemoResourceKeyVariable = "51DEGREES_RESOURCE_KEY"; + public const string DemoCloudEndpointVariable = "51DEGREES_CLOUD_ENDPOINT"; private const string _rootUrl = "CLOUD_ROOT_URL"; private const string _freeResourceKey = "FREE_RESOURCE_KEY"; private const string _paidResourceKey = "PAID_RESOURCE_KEY"; From e1a9d70236feac6f4968c85071a8c27bb2885b4e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 15 Sep 2026 06:42:33 +0100 Subject: [PATCH 3/7] TEST: Describe the platform's client script check as a tag check, without naming a person --- Browser51Did/PlatformAddsClientScriptTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Browser51Did/PlatformAddsClientScriptTests.cs b/Browser51Did/PlatformAddsClientScriptTests.cs index edf9dc8..bdf2fe0 100644 --- a/Browser51Did/PlatformAddsClientScriptTests.cs +++ b/Browser51Did/PlatformAddsClientScriptTests.cs @@ -11,10 +11,10 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// A page carrying the preference platform's tag and no client script tag /// at all. /// -/// James Rosewell, 15 September 2026. The platform has one route to the -/// third party cookie result and to whether the regulation applies, which -/// is the client script's object. Where that object is not on the page the -/// platform adds the script itself, using the cloud that served it and the +/// The platform has one route to the third party cookie result and to +/// whether the regulation applies, which is the client script's object. +/// Where the page has no client script tag the platform adds the script +/// itself, using the cloud that served it and the /// resource key it already holds, and says in the console that it did. A /// second route would be a second answer to the same question, and a /// publisher who forgot the tag would get a dialog behaving differently From 69499f08ab93ba9edbc8c860c1332c786415b9aa Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 15 Sep 2026 08:24:04 +0100 Subject: [PATCH 4/7] TEST: Read the resource key from the agreed names and launch the demo from pmp-web The 51Did browser tests read 51DEGREES_RESOURCE_KEY first and, where that is unset, _51DEGREES_RESOURCE_KEY_51DID, the name continuous integration sets for a resource key carrying the 51Did product. The demo is still given the key as 51DEGREES_RESOURCE_KEY. A missing key now names both variables in the skip reason, and both READMEs list the second name. The dotnet demo moved from Examples/Cloud/PreferenceManagement-Web to Examples/Cloud/pmp-web, so the launcher and the README point there. --- Browser51Did/Demo.cs | 5 +++-- Browser51Did/Harness.cs | 17 ++++++++++++----- Browser51Did/README.md | 12 +++++++----- Examples/ExampleApps.cs | 5 +++-- README.md | 3 ++- TestsCommon/TestConfig.cs | 34 +++++++++++++++++++++++++++++++--- 6 files changed, 58 insertions(+), 18 deletions(-) diff --git a/Browser51Did/Demo.cs b/Browser51Did/Demo.cs index c11cd40..03a7eac 100644 --- a/Browser51Did/Demo.cs +++ b/Browser51Did/Demo.cs @@ -100,8 +100,9 @@ public string JsonEndpointPath(ExampleDescriptor? descriptor) /// sibling checkout through , the way /// the Contract tests launch an example. It is started with exactly the two /// input variables every language's demo reads, 51DEGREES_RESOURCE_KEY and -/// 51DEGREES_CLOUD_ENDPOINT, taken from this run's own environment, and -/// with the runtime's own way of choosing a port. DEMO_MODE chooses the +/// 51DEGREES_CLOUD_ENDPOINT, taken from this run's own environment (the key +/// from _51DEGREES_RESOURCE_KEY_51DID where 51DEGREES_RESOURCE_KEY is +/// unset), and with the runtime's own way of choosing a port. DEMO_MODE chooses the /// route prefix, cloud unless pipeline is named. /// /// diff --git a/Browser51Did/Harness.cs b/Browser51Did/Harness.cs index 9c63491..b067ca0 100644 --- a/Browser51Did/Harness.cs +++ b/Browser51Did/Harness.cs @@ -81,7 +81,10 @@ public static string? CloudUrl /// /// The resource key the demo is started with, from - /// 51DEGREES_RESOURCE_KEY, the variable every language's demo reads. + /// 51DEGREES_RESOURCE_KEY, the variable every language's demo reads + /// first, or where that is unset from _51DEGREES_RESOURCE_KEY_51DID, + /// the name continuous integration sets. The demo is given it as + /// 51DEGREES_RESOURCE_KEY either way. /// /// The tests that create a 51Did for a standard or personalized answer /// need a resource key whose products include CloudV5FODiD. @@ -113,9 +116,11 @@ public static string? CloudUrl string.IsNullOrEmpty(Resource) ? ResourceKey.Missing : null, }.Where(missing => missing != null)) + " 51DEGREES_CLOUD_ENDPOINT names the cloud including its api/v4 " - + "path, and 51DEGREES_RESOURCE_KEY a resource key the cloud " + + "path, and 51DEGREES_RESOURCE_KEY, or where that is unset " + + "_51DEGREES_RESOURCE_KEY_51DID, a resource key the cloud " + "creates 51Dids for standard and personalized answers with. Both " - + "are handed to the demo unchanged. See Browser51Did/README.md."; + + "are handed to the demo under the first names. See " + + "Browser51Did/README.md."; /// /// The object name a page uses when it does not ask for another one, @@ -422,8 +427,10 @@ public static void RequireMarketingIdentifiers() + "personalized both are, so nothing here would be " + "testing the thing it is for. The service said: " + Redacted(_marketingRefusal) - + " Point 51DEGREES_RESOURCE_KEY at a resource key " - + "the service creates standard identifiers for."); + + " Point 51DEGREES_RESOURCE_KEY, or in continuous " + + "integration _51DEGREES_RESOURCE_KEY_51DID, at a " + + "resource key the service creates standard identifiers " + + "for."); } } diff --git a/Browser51Did/README.md b/Browser51Did/README.md index e9986b1..e639af2 100644 --- a/Browser51Did/README.md +++ b/Browser51Did/README.md @@ -6,7 +6,7 @@ only from an answer the visitor actually gave, with the 51Degrees Preference Management Platform (PMP) and the client script on a publisher's page. They drive a **demo**, a small web app that serves every page the tests -load. The dotnet demo is `Examples/Cloud/PreferenceManagement-Web` in +load. The dotnet demo is `Examples/Cloud/pmp-web` in [device-detection-dotnet-examples](https://github.com/51Degrees/device-detection-dotnet-examples). The pages, the recorder that watches them, the change watcher and the stub consent platform are plain static files in that demo, so a demo in another @@ -73,7 +73,8 @@ the Contract tests start an example. | Variable | Meaning | | --- | --- | | `51DEGREES_CLOUD_ENDPOINT` | The cloud, including its `api/v4` path. Handed to the demo unchanged. | -| `51DEGREES_RESOURCE_KEY` | A resource key the cloud creates 51Dids for standard and personalized answers with. Handed to the demo unchanged, and never printed, because every message a test produces has it taken out. | +| `51DEGREES_RESOURCE_KEY` | A resource key the cloud creates 51Dids for standard and personalized answers with. Handed to the demo under this name, and never printed, because every message a test produces has it taken out. | +| `_51DEGREES_RESOURCE_KEY_51DID` | Read only where `51DEGREES_RESOURCE_KEY` is unset, and handed to the demo as `51DEGREES_RESOURCE_KEY`. This is the name continuous integration sets, for a resource key carrying the 51Did product. | | `DEMO_LANG` | The demo to launch from the sibling checkout. `dotnet` unless set. | | `DEMO_URL` | A demo already running, which is used instead of launching one. | | `DEMO_MODE` | `cloud` unless set to `pipeline`. | @@ -87,10 +88,11 @@ env CLOUD_ROOT_URL="http://localhost:5050/" \ ``` `env` is used because bash cannot export a variable whose name starts with a -digit. In PowerShell write `${env:51DEGREES_RESOURCE_KEY} = '...'`. +digit, which is also why the name continuous integration sets starts with an +underscore. In PowerShell write `${env:51DEGREES_RESOURCE_KEY} = '...'`. -With the two input variables unset every test reports inconclusive with the -reason. A resource key that cannot create reports inconclusive with the +With the endpoint or both resource key names unset every test reports +inconclusive with the reason. A resource key that cannot create reports inconclusive with the cloud's own refusal, and so does a cloud that will not hold a shared choice. The shared choice needs the browser to keep the cloud's `Secure` `SameSite=None` cookie, which both browsers did from plain HTTP on diff --git a/Examples/ExampleApps.cs b/Examples/ExampleApps.cs index ebd54d5..48bd5e7 100644 --- a/Examples/ExampleApps.cs +++ b/Examples/ExampleApps.cs @@ -121,7 +121,8 @@ public static bool TryCreateDemo( /// routes, and reads its input data from the same two variables, /// 51DEGREES_RESOURCE_KEY and 51DEGREES_CLOUD_ENDPOINT, so a demo in /// another language is added here and nothing else in the suite - /// changes. + /// changes. The suite always hands the key over under the runtime + /// name, whichever of its two names the suite read it from. /// public static readonly IReadOnlyDictionary Demos = new Dictionary @@ -131,7 +132,7 @@ public static bool TryCreateDemo( WorkingDir: Path.Combine( RepoPaths.SiblingsRoot, "device-detection-dotnet-examples", - "Examples", "Cloud", "PreferenceManagement-Web"), + "Examples", "Cloud", "pmp-web"), Command: "dotnet", Args: new[] { "run", "-c", "Release", "--no-launch-profile" }, ReadinessPath: "/cloud/common", diff --git a/README.md b/README.md index eab517a..7716441 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ and no keys are committed. | `ENTERPRISE_V4_LICENSE` | `CloudInternal` | License passed to the JS endpoint to unlock paid properties. | | `SELENIUM_URL` | optional | Selenium grid URL; omit for a local Chrome driver. | | `EXAMPLE_URL` / `EXAMPLE_LANG` | `Contract` | The example app to test (CI / local). | -| `51DEGREES_CLOUD_ENDPOINT` / `51DEGREES_RESOURCE_KEY` | `Browser51Did` | Handed unchanged to the demo, the same names every language's demo reads. | +| `51DEGREES_CLOUD_ENDPOINT` / `51DEGREES_RESOURCE_KEY` | `Browser51Did` | Handed to the demo under these names, the ones every language's demo reads first. | +| `_51DEGREES_RESOURCE_KEY_51DID` | `Browser51Did` | Read where `51DEGREES_RESOURCE_KEY` is unset. The name CI sets, for a resource key carrying the 51Did product. | | `DEMO_URL` / `DEMO_LANG` / `DEMO_MODE` | `Browser51Did` | The demo to test, and `cloud` or `pipeline` pages. | A missing variable only fails the test that reads it. diff --git a/TestsCommon/TestConfig.cs b/TestsCommon/TestConfig.cs index a231eeb..15be038 100644 --- a/TestsCommon/TestConfig.cs +++ b/TestsCommon/TestConfig.cs @@ -14,14 +14,24 @@ public class TestConfig public string PaidResourceKey => Require(_paidResourceKey); // Enterprise V4 license, passed to the JS endpoint to unlock paid properties. public string EnterpriseV4License => Require(_enterpriseV4License); - // Resource key a demo is started with. The same name is read by every - // language's demo, so the suite hands it over unchanged. - public string DemoResourceKey => Require(DemoResourceKeyVariable); + // Resource key a demo is started with, read from the runtime name + // first and from the CI name where that is unset. The demo is always + // given it under the runtime name, the one every language's demo + // reads first. + public string DemoResourceKey => + RequireFirst(DemoResourceKeyVariable, DemoResourceKeyCiVariable); // Cloud endpoint a demo is started with, including the api/v4 path. // Read by every language's demo under the same name. public string DemoCloudEndpoint => Require(DemoCloudEndpointVariable); + // The names follow the 51Degrees naming scheme for keys. The runtime + // name is what a developer sets and is read first. The CI name starts + // with an underscore, because a shell cannot export a name that + // starts with a digit, and ends with the product the key must carry, + // which for these tests is 51Did, because they create identifiers + // for standard and personalized answers. public const string DemoResourceKeyVariable = "51DEGREES_RESOURCE_KEY"; + public const string DemoResourceKeyCiVariable = "_51DEGREES_RESOURCE_KEY_51DID"; public const string DemoCloudEndpointVariable = "51DEGREES_CLOUD_ENDPOINT"; private const string _rootUrl = "CLOUD_ROOT_URL"; private const string _freeResourceKey = "FREE_RESOURCE_KEY"; @@ -56,5 +66,23 @@ private static string Require(string name) } return value; } + + // Reads the first of several names that is set, and fails naming all + // of them, in the order they are read, when none is. + private static string RequireFirst(params string[] names) + { + foreach (var name in names) + { + string value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrEmpty(value) == false) + { + return value; + } + } + throw new InvalidOperationException( + "Required environment variable '" + + string.Join("', or where that is unset '", names) + + "' is not set."); + } } } From 13a66926c8c05bb21f8f55aa15f3cf4c12d50ec1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 15 Sep 2026 10:33:32 +0100 Subject: [PATCH 5/7] DOC: Say what the pipeline mode proves now the demo serves its pages The section said every run uses /cloud/ until a demo serves the pipeline pages, and that the tests about the PMP adding a script prove the cloud's script in either mode. The dotnet demo serves /pipeline/ now, and in that mode NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates reads Visitor.ClientRequests, which looks on the pipeline's path while the PMP adds the cloud's script, so the section now says that. Step 2 of adding a demo names /pipeline/ too. --- Browser51Did/README.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/Browser51Did/README.md b/Browser51Did/README.md index e639af2..fdeab03 100644 --- a/Browser51Did/README.md +++ b/Browser51Did/README.md @@ -103,19 +103,25 @@ another name. 1. Copy the demo's static files and templates, and fill the same placeholders from the same two variables. -2. Serve every route under `/cloud/`, answering any host name, because the - tests load the pages as `site-a.localtest` and `site-b.localtest` on the - demo's port. +2. Serve every route under `/cloud/`, and under `/pipeline/` once the + language's own web integration serves the client script, answering any + host name, because the tests load the pages as `site-a.localtest` and + `site-b.localtest` on the demo's port. 3. Add an entry to `ExampleApps.Demos` saying how to launch it. 4. Run the category with `DEMO_LANG` set to that entry. -## What only the cloud mode proves today - -The `/pipeline/` mode needs the demo's own pipeline to render the user -prompt block, which needs a 51Did element in that pipeline and a JavaScript -builder carrying the new template. Until a demo serves those pages every -run uses `/cloud/`. Two things will need attention when it does. The PMP -recognises a client script tag only by the cloud's path, `/api/v4/.js`, -so on a pipeline page it may add the cloud's script beside the demo's own, -and the tests about the PMP adding a script prove the cloud's script in -either mode. +## What the pipeline mode proves today + +The dotnet demo serves every route under `/pipeline/` as well, and +`DEMO_MODE=pipeline` drives those pages. Their client script comes from the +demo's own pipeline, which renders the user prompt block only with the +pipeline packages that carry the new template, so with the released +packages those pages show no prompt block and the tests that press it fail +there. Two things differ in that mode. The PMP recognises a client script +tag only by the cloud's path, `/api/v4/.js`, so the demo's pipeline +pages carry a tag that is not `async`, which has always run before the PMP +looks for its object. And on a page with no client script tag the PMP adds +the cloud's script, which posts to the cloud's `/api/v4/json`, whilst +`Visitor.ClientRequests` looks on the pipeline's path, so +`NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates` fails in that +mode until the suite looks for the added script's own requests. From bf56d0eec47a03f7e025e193cf194f0966462d19 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 16 Sep 2026 00:50:26 +0100 Subject: [PATCH 6/7] FIX: Say PMP throughout, and drop a helper naming an address that is going away The suite called the Preference Management Platform "the platform" in prose and "Platform" in identifiers, against the rule that every name uses pmp and every sentence says PMP. Renamed the two test classes and their files, the Visitor and Harness members, the five test methods and the CommonPath parameter, and swept the prose. Left as they were, on purpose: 1. Routes.NoPlatform and Routes.PlatformOnly, because their values are the URL paths the demo serves and the constant should keep naming the URL it produces. Renaming the pair needs the demo's own page names to move in the same release. 2. The client script's console wording quoted in Harness, because NoPmpMessage is matched against that output as a substring and rewording it would break the match. 3. The consent platform tests, where a consent management platform is a different product from PMP. Harness.PlatformLoaderUrl() named {CloudUrl}/api/v4/pmp and nothing called it. That address answers 404 once the loader takes the resource key in its path, so the helper is removed rather than corrected. An address the suite does not ask for is better absent than wrong. Four comments pointed at another repository's internal paths and change numbers, which a reader here cannot follow, and one of them printed into the test output. Each now says what the thing is instead. The same goes for the references to an internal list of demonstrations. Build: dotnet build SeleniumApiTests.csproj -c Release, 0 warnings, 0 errors. --- Browser51Did/Browser51DidTestBase.cs | 2 +- .../ConsentPlatformAcceptanceTests.cs | 17 +++-- Browser51Did/Demo.cs | 10 +-- Browser51Did/Harness.cs | 24 +++---- Browser51Did/IsGdprAcceptanceTests.cs | 40 +++++------ ...ceptanceTests.cs => PmpAcceptanceTests.cs} | 68 +++++++++---------- ...ptTests.cs => PmpAddsClientScriptTests.cs} | 54 +++++++-------- Browser51Did/README.md | 4 +- Browser51Did/TcString.cs | 9 ++- Browser51Did/Visitor.cs | 39 ++++++----- 10 files changed, 131 insertions(+), 136 deletions(-) rename Browser51Did/{PlatformAcceptanceTests.cs => PmpAcceptanceTests.cs} (91%) rename Browser51Did/{PlatformAddsClientScriptTests.cs => PmpAddsClientScriptTests.cs} (82%) diff --git a/Browser51Did/Browser51DidTestBase.cs b/Browser51Did/Browser51DidTestBase.cs index 17dc049..86790ba 100644 --- a/Browser51Did/Browser51DidTestBase.cs +++ b/Browser51Did/Browser51DidTestBase.cs @@ -198,7 +198,7 @@ protected static FodId Parse(string identifier, string what) /// /// The identifier says the usage was stated by the caller, which is - /// what the preference platform does, so the signal source is direct. + /// what PMP does, so the signal source is direct. /// protected static void AssertDirect(string identifier, string what) { diff --git a/Browser51Did/ConsentPlatformAcceptanceTests.cs b/Browser51Did/ConsentPlatformAcceptanceTests.cs index 49e696c..6870981 100644 --- a/Browser51Did/ConsentPlatformAcceptanceTests.cs +++ b/Browser51Did/ConsentPlatformAcceptanceTests.cs @@ -8,10 +8,9 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// -/// A page with a consent platform on it and no preference platform, which -/// is the other half of demonstration 2 and the whole of demonstration 3's -/// second case, being the identifier whose usage was decoded from a -/// framework string rather than stated. +/// A page with a consent platform on it and no PMP, which is the case +/// where the identifier's usage was decoded from a framework string +/// rather than stated. /// /// The consent platform here is a stub, and that is the contract rather /// than a gap. The client script uses only what the framework's @@ -19,7 +18,7 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// addEventListener and a callback carrying tcString and eventStatus, so a /// stub honouring those is what every product has to do, and a product /// that breaks it is the product's defect. The two never share a page, so -/// nothing here also carries the preference platform. +/// nothing here also carries PMP. /// /// [TestClass, TestCategory("Browser51Did")] @@ -75,7 +74,7 @@ public void ConsentPlatformOnly_SecondRequestCarriesTheString() var second = visitor.ClientRequests().Last(); Assert.IsNotNull( second.Form("tcstring"), - "the framework string the platform delivered must reach the " + "the framework string PMP delivered must reach the " + $"cloud. The body was: {second.Body}"); // The demo's stub consent platform delivers one fixed string, and // every language's demo has to deliver the same one, so it is @@ -114,13 +113,13 @@ public void ConsentPlatformOnly_SecondRequestCarriesTheString() } /// - /// Demonstration 8. A page with nowhere to get an answer from creates + /// A page with nowhere to get an answer from creates /// nothing, says so once, and stores a record that carries no answer, /// so a later page view with an answer is a different input and asks /// again rather than reusing this one. /// [Browser51DidTest] - public void NoPlatformAtAll_NoIdentifierAndOneWarning() + public void NoPmpAtAll_NoIdentifierAndOneWarning() { using var visitor = NewVisitor(Chrome); visitor.Go(Harness.SiteA, Routes.NoPlatform); @@ -162,7 +161,7 @@ public void NoPlatformAtAll_NoIdentifierAndOneWarning() + "does have one would look the same as this and be served " + $"from the cache. The record was: {record}"); - var warnings = visitor.ConsoleMatching(Harness.NoPlatformMessage); + var warnings = visitor.ConsoleMatching(Harness.NoPmpMessage); Assert.AreEqual( 1, warnings.Count, diff --git a/Browser51Did/Demo.cs b/Browser51Did/Demo.cs index 03a7eac..82e5bec 100644 --- a/Browser51Did/Demo.cs +++ b/Browser51Did/Demo.cs @@ -17,13 +17,13 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// public static class Routes { - /// The platform, then the client script. + /// PMP, then the client script. public const string Common = "common"; - /// The client script, then the platform. + /// The client script, then PMP. public const string CommonScriptFirst = "common-script-first"; - /// The platform, the client script and the change watcher. + /// PMP, the client script and the change watcher. public const string Change = "change"; /// The first of two pages carrying what does. @@ -38,11 +38,11 @@ public static class Routes /// The client script alone. public const string NoPlatform = "no-platform"; - /// The platform alone, with no client script tag. + /// PMP alone, with no client script tag. public const string PlatformOnly = "platform-only"; /// - /// The platform alone, naming the client script's object + /// PMP alone, naming the client script's object /// . /// public const string NamedObject = "named-object"; diff --git a/Browser51Did/Harness.cs b/Browser51Did/Harness.cs index b067ca0..c6e1cdc 100644 --- a/Browser51Did/Harness.cs +++ b/Browser51Did/Harness.cs @@ -25,7 +25,7 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// /// The thing under test is a demo, a web app serving the pages these tests /// load, which every language mirrors, started with a cloud endpoint and a -/// resource key. The preference platform and the shared store come from +/// resource key. PMP and the shared store come from /// that cloud whichever demo is chosen. See . /// /// @@ -124,14 +124,14 @@ public static string? CloudUrl /// /// The object name a page uses when it does not ask for another one, - /// which is what the client script and the preference platform both + /// which is what the client script and PMP both /// fall back to. /// public const string DefaultObjectName = "fod"; /// /// The object name the demo's page - /// gives the preference platform in data-object-name. + /// gives PMP in data-object-name. /// public const string NamedObject = "fiftyOneData"; @@ -176,16 +176,14 @@ public static string? CloudUrl /// under test carries it, which is what makes a green run mean /// something. /// - /// Settled by the template work package. If that wording changes, this - /// is the one place to change it, and the report for that package - /// carries the string in force. + /// This is the one place to change if that wording changes. /// /// public const string IterationLimitMessage = "51Degrees: the maximum of"; /// /// Text that exists only inside the template's user prompt section, - /// being the name the preference platform puts its own surface under. + /// being the name PMP puts its own surface under. /// The iteration limit message above sits outside that section, so it /// says the template is the new one and says nothing about whether the /// block was rendered. This says the block is there. @@ -202,7 +200,7 @@ public static string? CloudUrl /// the sentence may change, and the test using it also asserts that no /// value was printed beside it. /// - public const string NoPlatformMessage = + public const string NoPmpMessage = "no preference platform was found"; /// @@ -218,8 +216,8 @@ public static string? CloudUrl "already exists on this page"; /// - /// What the preference platform says when it finds no client script - /// object on the page. Read from the platform's own source, where the + /// What PMP says when it finds no client script + /// object on the page. Read from PMP's own source, where the /// sentence is "There is no client script object named '{name}' on /// this page, so the client script is being added from the cloud that /// served this one." @@ -236,8 +234,6 @@ public static string? CloudUrl #endregion - /// The preference platform's loader, as the page asks for it. - public static string PlatformLoaderUrl() => $"{CloudUrl}/api/v4/pmp"; /// /// Server to server, with certificate validation relaxed because the @@ -266,8 +262,8 @@ public static string? CloudUrl /// Two markers, because they say different things. The iteration limit /// message says the template is the new one. It sits outside the user /// prompt section, so it is in the rendered script whenever updates - /// are enabled, whether or not the block itself was rendered. The - /// platform's own global name appears only inside the section, so it + /// are enabled, whether or not the block itself was rendered. PMP's + /// own global name appears only inside the section, so it /// is what says the block is actually there. A run that had only the /// first would be testing the new template with the block switched /// off, which passes nothing it is meant to prove. diff --git a/Browser51Did/IsGdprAcceptanceTests.cs b/Browser51Did/IsGdprAcceptanceTests.cs index be036a2..2a298aa 100644 --- a/Browser51Did/IsGdprAcceptanceTests.cs +++ b/Browser51Did/IsGdprAcceptanceTests.cs @@ -7,20 +7,21 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// -/// Whether the regulation applies, and where the preference platform gets +/// Whether the regulation applies, and where PMP gets /// that from. /// -/// **These cannot pass until the change that puts IsGdpr in the cloud is -/// released**, being pipeline-dotnet pull request 413 and then cloud pull -/// request 372. The harness entitlement record already asks for the -/// property, so the moment the container carries it these run. Until then +/// **These cannot pass until the change that puts IsGdpr in the cloud +/// service is released**, being pipeline-dotnet pull request 413 and then +/// a cloud service release that carries it. The harness entitlement +/// record already asks for the property, so the moment the container +/// carries it these run. Until then /// the first test reports inconclusive with that reason rather than /// failing, because a red test nobody can fix teaches a reader to ignore /// red tests. /// /// /// The dialog is shown and an identifier created either way, because the -/// question the platform asks is the Model Terms usage, which is a matter +/// question PMP asks is the Model Terms usage, which is a matter /// of contract, and not a consent under the regulation. /// /// @@ -28,7 +29,7 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; public class IsGdprAcceptanceTests : Browser51DidTestBase { /// - /// With the property carrying a value, the platform's framework + /// With the property carrying a value, PMP's framework /// surface reports it, and a false value does not stop the visitor /// being asked. /// @@ -37,7 +38,7 @@ public void IsGdpr_ReadFromTheClientScript_SetsGdprApplies() { using var visitor = NewVisitor(Chrome); visitor.Go(Harness.SiteA, Routes.Common); - visitor.WaitForPlatform(); + visitor.WaitForPmp(); visitor.WaitForClientRounds(1); var value = visitor.IsGdpr(); @@ -45,18 +46,19 @@ public void IsGdpr_ReadFromTheClientScript_SetsGdprApplies() { Assert.Inconclusive( "The client script's object carries no isgdpr value, so " - + "there is nothing for the platform to read. The property " - + "reaches the cloud in pipeline-dotnet pull request 413 " - + "and then cloud pull request 372, and the harness " - + "entitlement record already asks for it, so this runs as " - + "soon as the container carries it."); + + "there is nothing for the PMP to read. The property " + + "reaches the cloud service in pipeline-dotnet pull " + + "request 413 and then a cloud service release that " + + "carries it, and the harness entitlement record already " + + "asks for it, so this runs as soon as the container " + + "carries it."); } var applies = visitor.GdprApplies(); Assert.AreEqual( string.Equals(value, "True", StringComparison.OrdinalIgnoreCase), applies, - "the platform's framework surface must report what the client " + "PMP's framework surface must report what the client " + $"script resolved. The script said '{value}' and the surface " + $"said {applies}."); @@ -76,15 +78,15 @@ public void IsGdpr_ReadFromTheClientScript_SetsGdprApplies() } /// - /// With no value to read, the platform says so once and carries on as + /// With no value to read, PMP says so once and carries on as /// though the regulation applies, which is the safe way round. /// [Browser51DidTest] - public void IsGdprAbsent_PlatformWarnsAndAssumesItApplies() + public void IsGdprAbsent_PmpWarnsAndAssumesItApplies() { using var visitor = NewVisitor(Chrome); visitor.Go(Harness.SiteA, Routes.Common); - visitor.WaitForPlatform(); + visitor.WaitForPmp(); visitor.WaitForClientRounds(1); if (visitor.IsGdpr() != "") @@ -98,11 +100,11 @@ public void IsGdprAbsent_PlatformWarnsAndAssumesItApplies() Assert.IsTrue( visitor.GdprApplies(), - "with nothing to read, the platform assumes the regulation " + "with nothing to read, PMP assumes the regulation " + "applies rather than assuming it does not."); Assert.IsTrue( visitor.ConsoleMatching("isgdpr").Count > 0, - "the platform says once that it could not read the property, " + "PMP says once that it could not read the property, " + "because a publisher whose key does not carry it has no other " + "way of finding out. The console said: " + string.Join(" | ", visitor.Console())); diff --git a/Browser51Did/PlatformAcceptanceTests.cs b/Browser51Did/PmpAcceptanceTests.cs similarity index 91% rename from Browser51Did/PlatformAcceptanceTests.cs rename to Browser51Did/PmpAcceptanceTests.cs index ebcd3ab..6564e37 100644 --- a/Browser51Did/PlatformAcceptanceTests.cs +++ b/Browser51Did/PmpAcceptanceTests.cs @@ -8,20 +8,21 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// -/// The preference platform and the client script on one page, which is the +/// PMP and the client script on one page, which is the /// arrangement the design is built around, proved in a real browser. /// -/// These are demonstrations 1, 3 and 4 of the acceptance list, with the -/// change of answer and the alternative answer that came from the later -/// decisions. Every assertion is on an ordering of requests and page -/// states, never on a clock. +/// These cover the shared choice read on a second site, the first visit +/// that runs the whole sequence, and the two tags in either order, with +/// the change of answer and the alternative answer alongside them. Every +/// assertion is on an ordering of requests and page states, never on a +/// clock. /// /// [TestClass, TestCategory("Browser51Did")] -public class PlatformAcceptanceTests : Browser51DidTestBase +public class PmpAcceptanceTests : Browser51DidTestBase { /// - /// Demonstration 3, and the one that carries the most. A first visit, + /// The test that carries the most. A first visit, /// the visitor answers the first card, the answer reaches the client /// script, the full sequence runs with the usage known and an /// identifier comes back with the signal source recorded as direct. @@ -29,36 +30,35 @@ public class PlatformAcceptanceTests : Browser51DidTestBase /// waits between the two cards. /// [Browser51DidTest] - public void CommonPath_PlatformThenScript_Chrome() - => CommonPath(Chrome, platformFirst: true); + public void CommonPath_PmpThenScript_Chrome() + => CommonPath(Chrome, pmpFirst: true); /// - /// The same, in the other browser. Demonstration 9 asks for both, - /// because anything the shared choice travels on behaves differently - /// between them. + /// The same, in the other browser. Both are run because anything the + /// shared choice travels on behaves differently between them. /// [Browser51DidTest] - public void CommonPath_PlatformThenScript_Firefox() - => CommonPath(Firefox, platformFirst: true); + public void CommonPath_PmpThenScript_Firefox() + => CommonPath(Firefox, pmpFirst: true); /// - /// The same page with the two tags the other way round. The platform's + /// The same page with the two tags the other way round. PMP's /// bundle loads on its own timetable, so the announcement has to reach /// the client script whichever tag the publisher wrote first, and a /// design that only worked one way round would pass the test above and /// fail on half the customers' pages. /// [Browser51DidTest] - public void CommonPath_ScriptThenPlatform_Chrome() - => CommonPath(Chrome, platformFirst: false); + public void CommonPath_ScriptThenPmp_Chrome() + => CommonPath(Chrome, pmpFirst: false); - private void CommonPath(string browser, bool platformFirst) + private void CommonPath(string browser, bool pmpFirst) { RequireMarketingIdentifiers(); using var visitor = NewVisitor(browser); visitor.Go( Harness.SiteA, - platformFirst ? Routes.Common : Routes.CommonScriptFirst); + pmpFirst ? Routes.Common : Routes.CommonScriptFirst); // The first round. Nobody has been asked yet, so nothing may be // created, which is the rule the whole programme exists for. @@ -81,7 +81,7 @@ private void CommonPath(string browser, bool platformFirst) + "resolved."); // The visitor answers. - visitor.WaitForPlatform(); + visitor.WaitForPmp(); visitor.WaitForCard("preferences"); // Records when each round finishes and when the share card arrives, // for the failure message below. It changes nothing on the page. @@ -152,7 +152,7 @@ private void CommonPath(string browser, bool platformFirst) var identifier = visitor.Identifier(); AssertDirect( identifier, - "an answer given on the platform is stated directly"); + "an answer given on PMP is stated directly"); Assert.AreEqual( Usage.Standard, UsageOf(identifier, "the answer was standard"), @@ -188,7 +188,7 @@ private void CommonPath(string browser, bool platformFirst) } /// - /// Demonstration 1. A choice made on one site is read on another, the + /// A choice made on one site is read on another, the /// visitor is not asked again, and the client script still creates an /// identifier from the answer with the signal source recorded as /// direct. @@ -227,14 +227,14 @@ private void SharedChoice(string browser) // Site B, a first visit, in the same browser. visitor.Go(Harness.SiteB, Routes.Common); - visitor.WaitForPlatform(); + visitor.WaitForPmp(); visitor.WaitForClientRounds(1); var read = visitor.SharedStoreRequests() .FirstOrDefault(r => r.Method == "GET" && r.Done); Assert.IsNotNull( read, - "the platform must ask the shared store what this visitor " + "PMP must ask the shared store what this visitor " + "already chose. It made no such call. The requests were: " + string.Join(" || ", visitor.Requests().Select(r => r.ToString()))); @@ -246,7 +246,7 @@ private void SharedChoice(string browser) Assert.IsTrue( visitor.BubbleOnly(), "a visitor who has already answered is not asked again, so only " - + $"the floating button shows. The dialog is: {visitor.PlatformState()}" + + $"the floating button shows. The dialog is: {visitor.PmpState()}" + ". The console said: " + string.Join(" | ", visitor.Console())); @@ -267,7 +267,7 @@ private void SharedChoice(string browser) var identifier = visitor.Identifier(); AssertDirect( identifier, - "a choice made on the platform stays a stated usage on the " + "a choice made on PMP stays a stated usage on the " + "second site"); Assert.AreEqual( Usage.Standard, @@ -276,15 +276,15 @@ private void SharedChoice(string browser) Assert.AreEqual( "standard", - visitor.PlatformPreference(), - "the platform's own getter must answer with the choice it is " + visitor.PmpPreference(), + "PMP's own getter must answer with the choice it is " + "acting on, on the second site as much as on the first."); var stored = visitor.LocalStorageKeys(); Assert.AreEqual( 0, stored.Count, - "nothing new is written to browser storage by the platform, and " + "nothing new is written to browser storage by PMP, and " + "the second site's answer lives in the shared store rather " + "than being copied here. It wrote: " + string.Join(", ", stored)); @@ -311,8 +311,8 @@ public void ChangeOfAnswer_SamePage_NewIdentifierAndOnChange() var roundsBefore = visitor.ClientRequests().Count; var sequenceBefore = Sequence(visitor.ClientRequests().Last()); - // The visitor changes their mind, through the platform. - visitor.OpenPlatform(); + // The visitor changes their mind, through PMP. + visitor.OpenPmp(); visitor.WaitForCard("preferences"); visitor.Press("personalized"); @@ -397,7 +397,7 @@ public void ChangeOfAnswer_AcrossPages_TheSecondPageCarriesTheNewAnswer() "the visitor pressed standard first."); // The answer is changed before the visitor leaves the first page. - visitor.OpenPlatform(); + visitor.OpenPmp(); visitor.WaitForCard("preferences"); visitor.Press("personalized"); Harness.Until( @@ -497,8 +497,8 @@ public void AlternativeAnswer_CreatesNonMarketingAndFiresTheAction() // it and what the request carried is on the record. Assert.AreEqual( "non-marketing", - visitor.PlatformPreference(), - "the platform is holding an answer even though its framework " + visitor.PmpPreference(), + "PMP is holding an answer even though its framework " + "surface reports none, and the answer is what creates the " + "identifier."); } diff --git a/Browser51Did/PlatformAddsClientScriptTests.cs b/Browser51Did/PmpAddsClientScriptTests.cs similarity index 82% rename from Browser51Did/PlatformAddsClientScriptTests.cs rename to Browser51Did/PmpAddsClientScriptTests.cs index bdf2fe0..1e8fd6c 100644 --- a/Browser51Did/PlatformAddsClientScriptTests.cs +++ b/Browser51Did/PmpAddsClientScriptTests.cs @@ -8,12 +8,12 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// -/// A page carrying the preference platform's tag and no client script tag +/// A page carrying PMP's tag and no client script tag /// at all. /// -/// The platform has one route to the third party cookie result and to +/// PMP has one route to the third party cookie result and to /// whether the regulation applies, which is the client script's object. -/// Where the page has no client script tag the platform adds the script +/// Where the page has no client script tag PMP adds the script /// itself, using the cloud that served it and the /// resource key it already holds, and says in the console that it did. A /// second route would be a second answer to the same question, and a @@ -22,42 +22,42 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// /// [TestClass, TestCategory("Browser51Did")] -public class PlatformAddsClientScriptTests : Browser51DidTestBase +public class PmpAddsClientScriptTests : Browser51DidTestBase { /// /// The whole of it in one page view, from the script arriving to the /// identifier coming back. /// [Browser51DidTest] - public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() + public void NoClientScriptTag_PmpAddsItAndTheAnswerStillCreates() { RequireMarketingIdentifiers(); using var visitor = NewVisitor(Chrome); visitor.Go(Harness.SiteA, Routes.PlatformOnly); - visitor.WaitForPlatform(); + visitor.WaitForPmp(); - // The script the platform added, named by where it came from and + // The script PMP added, named by where it came from and // which publisher it is for. Harness.Until( () => AddedClientScript(visitor) != null, - "the platform added the client script. The console said: " + "PMP added the client script. The console said: " + string.Join(" | ", visitor.Console())); var added = AddedClientScript(visitor)!; Assert.IsTrue( added.StartsWith(Harness.CloudUrl!, StringComparison.OrdinalIgnoreCase), - "the script must come from the cloud that served the platform, " - + "because that is the only cloud the platform knows about. It " + "the script must come from the cloud that served PMP, " + + "because that is the only cloud PMP knows about. It " + $"came from {added}."); Assert.IsTrue( added.Contains(Harness.Resource!, StringComparison.Ordinal), - "the script must be asked for with the resource key the " - + "platform already holds, or the cloud has no idea which " + "the script must be asked for with the resource key PMP " + + "already holds, or the cloud has no idea which " + $"publisher is asking. The URL was {added}."); // And it said so, which is the point of the convenience. Assert.IsTrue( visitor.ConsoleMatching(Harness.NoClientScriptMessage).Count > 0, - "the platform must say in the console that there was no client " + "PMP must say in the console that there was no client " + "script object on the page, because a publisher who left the " + "tag out has no other way of finding out. The console said: " + string.Join(" | ", visitor.Console())); @@ -71,10 +71,10 @@ public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() visitor.WaitForClientObject(); Assert.IsTrue( visitor.HasTheNewClientScript(), - "the script the platform added must be the new one, or the " - + "platform has quietly given itself the old behaviour."); + "the script PMP added must be the new one, or PMP " + + "has quietly given itself the old behaviour."); - // The third party cookie result reached the platform through it, + // The third party cookie result reached PMP through it, // which is the reason the script is added at all. Harness.Until( () => visitor.ThirdPartyCookies() != "", @@ -88,7 +88,7 @@ public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() Harness.Until( () => visitor.ClientRequests() .Any(r => r.Done && r.Form("id.usage") == "standard"), - "the answer reached the cloud through the script the platform " + "the answer reached the cloud through the script PMP " + "added. The console said: " + string.Join(" | ", visitor.Console())); var answered = visitor.ClientRequests() @@ -112,18 +112,18 @@ public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() + string.Join(" | ", visitor.Console())); AssertDirect( visitor.Identifier(), - "an answer given on the platform is stated directly"); + "an answer given on PMP is stated directly"); Assert.AreEqual( Usage.Standard, UsageOf(visitor.Identifier(), "the answer"), "the visitor pressed standard."); - // One script, not two. The platform adds one only where there is + // One script, not two. PMP adds one only where there is // none, so nothing here may trip the warning about a second copy. Assert.AreEqual( 0, visitor.ConsoleMatching(Harness.SecondInstanceWarning).Count, - "the platform added a second copy of the client script, or " + "PMP added a second copy of the client script, or " + "added one to a page that already had it. The console said: " + string.Join(" | ", visitor.Console())); Assert.AreEqual( @@ -135,7 +135,7 @@ public void NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates() /// /// The publisher may name the object something other than the default, - /// and the platform then asks for the script under that name, finds it + /// and PMP then asks for the script under that name, finds it /// under that name, and says which name it used. /// [Browser51DidTest] @@ -144,18 +144,18 @@ public void ObjectNameAttribute_NamesTheObjectEverywhere() const string objectName = Harness.NamedObject; using var visitor = NewVisitor(Chrome); visitor.Go(Harness.SiteA, Routes.NamedObject); - visitor.WaitForPlatform(); + visitor.WaitForPmp(); Harness.Until( () => AddedClientScript(visitor) != null, - "the platform added the client script. The console said: " + "PMP added the client script. The console said: " + string.Join(" | ", visitor.Console())); var added = AddedClientScript(visitor)!; Assert.IsTrue( added.Contains(objectName, StringComparison.Ordinal), "the name the publisher asked for must be on the URL, or the " - + "cloud renders the script under the default name and the " - + $"platform then looks for the wrong object. The URL was {added}."); + + "cloud renders the script under the default name and PMP " + + $"then looks for the wrong object. The URL was {added}."); visitor.WaitForClientObject(objectName); Assert.IsTrue( @@ -185,7 +185,7 @@ public void ObjectNameAttributeAbsent_UsesTheDefaultName() { using var visitor = NewVisitor(Chrome); visitor.Go(Harness.SiteA, Routes.PlatformOnly); - visitor.WaitForPlatform(); + visitor.WaitForPmp(); visitor.WaitForClientObject(Harness.DefaultObjectName); Assert.IsTrue( visitor.HasTheNewClientScript(Harness.DefaultObjectName), @@ -193,7 +193,7 @@ public void ObjectNameAttributeAbsent_UsesTheDefaultName() } /// - /// The client script the platform added, or null where it has not + /// The client script PMP added, or null where it has not /// added one yet. Anything the cloud serves as a resource key script /// counts, and the page carried none of its own, so whatever is there /// was added. diff --git a/Browser51Did/README.md b/Browser51Did/README.md index fdeab03..c4b87a2 100644 --- a/Browser51Did/README.md +++ b/Browser51Did/README.md @@ -54,7 +54,7 @@ user prompt block. | `change` | PMP, client script, change watcher | change of answer on one page | | `two/one`, `two/two` | the same as `change` | change of answer across pages | | `consent` | stub consent platform, then client script | consent platform only | -| `no-platform` | client script alone | no platform at all | +| `no-platform` | client script alone | no PMP at all | | `platform-only` | PMP alone | PMP adds the client script, default object name | | `named-object` | PMP with `data-object-name="fiftyOneData"` | object name attribute | @@ -123,5 +123,5 @@ pages carry a tag that is not `async`, which has always run before the PMP looks for its object. And on a page with no client script tag the PMP adds the cloud's script, which posts to the cloud's `/api/v4/json`, whilst `Visitor.ClientRequests` looks on the pipeline's path, so -`NoClientScriptTag_PlatformAddsItAndTheAnswerStillCreates` fails in that +`NoClientScriptTag_PmpAddsItAndTheAnswerStillCreates` fails in that mode until the suite looks for the added script's own requests. diff --git a/Browser51Did/TcString.cs b/Browser51Did/TcString.cs index 44ea808..7353c4f 100644 --- a/Browser51Did/TcString.cs +++ b/Browser51Did/TcString.cs @@ -11,11 +11,10 @@ namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; /// are set, because those are the only ones the server reads when it works /// out a usage from a string. /// -/// This is the same construction as TcStringBuilder in the cloud -/// repository's Did/Tests/FiftyOne.Did.OnPremise.Tests. It is -/// repeated here because that is another repository, and because a browser -/// test that quietly changed when a unit test helper changed would be worse -/// than a small repetition. +/// This is the same construction the cloud service's own unit tests use. +/// It is repeated here rather than shared, because a browser test that +/// quietly changed when a unit test helper changed would be worse than a +/// small repetition. /// /// /// Every language's demo delivers from its stub diff --git a/Browser51Did/Visitor.cs b/Browser51Did/Visitor.cs index ffe696a..d21fd0d 100644 --- a/Browser51Did/Visitor.cs +++ b/Browser51Did/Visitor.cs @@ -75,7 +75,7 @@ public sealed class Visitor : IDisposable }; /// - /// Finding the preference platform's dialog. It renders into a shadow + /// Finding PMP's dialog. It renders into a shadow /// root attached to a plain div with no name of its own, so the root /// is found by what is inside it. /// @@ -169,7 +169,7 @@ public IReadOnlyList ClientRequests() .ToList(); /// - /// The preference platform's calls to the shared store, being the read + /// PMP's calls to the shared store, being the read /// on load and the write when a visitor agrees to share. /// public IReadOnlyList SharedStoreRequests() @@ -200,7 +200,7 @@ public IReadOnlyList ConsoleMatching(string text) .Where(line => line.Contains(text, StringComparison.OrdinalIgnoreCase)) .ToList(); - /// The preferences the platform's action URL was fired with. + /// The preferences PMP's action URL was fired with. public IReadOnlyList Actions() => JsonSerializer.Deserialize>( Read("return JSON.stringify(" @@ -372,10 +372,10 @@ public IReadOnlyList ScriptSources() #endregion - #region The preference platform + #region PMP - /// Whether the platform has put anything on the page yet. - public bool PlatformLoaded() => Script("return pmpRoot() !== null;"); + /// Whether PMP has put anything on the page yet. + public bool PmpLoaded() => Script("return pmpRoot() !== null;"); /// Whether a named card is on the page and visible. public bool CardVisible(string card) @@ -394,11 +394,10 @@ public bool CardVisible(string card) /// /// /// The button is found by what it does rather than by its class, - /// because the build renames every class in the stylesheet to one or - /// two letters (pmp/dist/class-map.json turns pmp-fab into y and - /// pmp-popup into ag), so a test written against the class names in - /// the source finds nothing in the bundle a publisher is actually - /// served. The data attributes are not renamed. + /// because the PMP build renames every class in the stylesheet to one + /// or two letters, so a test written against the class names in the + /// source finds nothing in the bundle a publisher is actually served. + /// The data attributes are not renamed. /// /// public bool BubbleOnly() @@ -419,10 +418,10 @@ public bool BubbleOnly() /// message. A test that says only that something was not visible /// leaves the next person to open a browser by hand and find out why. /// - public string PlatformState() + public string PmpState() => Script(@" var root = pmpRoot(); - if (!root) { return 'the platform has rendered nothing'; } + if (!root) { return 'the PMP has rendered nothing'; } function describe(el, name) { if (!el) { return name + '=absent'; } var box = el.getBoundingClientRect(); @@ -469,10 +468,10 @@ public void Press(string action) } /// - /// The answer the platform is holding, through the getter it exposes + /// The answer PMP is holding, through the getter it exposes /// for a publisher. Empty where it holds none. /// - public string PlatformPreference() + public string PmpPreference() => Script(@" var api = window.__51d_pmp; if (!api || typeof api.preference !== 'function') { return ''; } @@ -481,7 +480,7 @@ public string PlatformPreference() ?? ""; /// Opens the dialog again, as a publisher's own link would. - public void OpenPlatform() + public void OpenPmp() => Script("window.__51d_pmp.open(); return null;"); /// @@ -652,11 +651,11 @@ public string Timeline() #endregion - /// Waits until the platform has put its dialog on the page. - public void WaitForPlatform() + /// Waits until PMP has put its dialog on the page. + public void WaitForPmp() => Harness.Until( - PlatformLoaded, - $"the preference platform loaded in {BrowserName}. " + PmpLoaded, + $"PMP loaded in {BrowserName}. " + $"{ServedSoFar()}. The console said: " + $"{string.Join(" | ", Console())}"); From 25ac75414735d23b429d8f1af290b275529c6b5e Mon Sep 17 00:00:00 2001 From: Oleksandr Lazarenko Date: Wed, 16 Sep 2026 10:47:09 -0400 Subject: [PATCH 7/7] BUG: Redact the resource key where MSTest reports it The redaction attribute edited the TestResult MSTest had already built. Assigning TestFailureException on a result that already has one does not replace it, MSTest aggregates, and it has already copied the first message into an internal member this assembly cannot set. A failing test therefore reported "One or more errors occurred. () ()" - the resource key and the redaction of it side by side, in a public repository's public build log. The attribute now returns a new TestResult, which has no first exception to aggregate with and no such member. DisplayName is redacted too, so the rule is that nothing on a reported result carries the key, with no exceptions to remember. Harness.Around spliced 160 raw bytes of the cloud-rendered client script into the guard's evidence. That evidence is written from ClassInitialize and so never passed through the attribute at all. It now redacts inside itself rather than at its one call site, because a caller cannot tell from the returned text whether it needs it. Harness.Resource is read once when the class is first touched, so no test can set it. Each redacting method gained an internal overload taking the key as an argument, with the public form delegating to it. Browser51Did/RedactionTests.cs covers both, six tests needing no cloud, demo or browser. Three of the six fail against the pre-fix code. --- Browser51Did/Browser51DidTestBase.cs | 79 +++++++-- Browser51Did/Harness.cs | 40 ++++- Browser51Did/README.md | 10 +- Browser51Did/RedactionTests.cs | 237 +++++++++++++++++++++++++++ 4 files changed, 340 insertions(+), 26 deletions(-) create mode 100644 Browser51Did/RedactionTests.cs diff --git a/Browser51Did/Browser51DidTestBase.cs b/Browser51Did/Browser51DidTestBase.cs index 86790ba..278f4fd 100644 --- a/Browser51Did/Browser51DidTestBase.cs +++ b/Browser51Did/Browser51DidTestBase.cs @@ -39,39 +39,82 @@ public override async Task ExecuteAsync( { var results = await base.ExecuteAsync(testMethod) .ConfigureAwait(false); - foreach (var result in results) + for (var index = 0; index < results.Length; index++) { - Redact(result); + results[index] = Redacted(results[index]); } return results; } - private static void Redact(TestResult result) + /// + /// A fresh result carrying everything the runner reports, with the + /// resource key taken out of it. + /// + /// A new result rather than an edit of the one MSTest built, because + /// setting on a result + /// that already has one does not replace it: MSTest keeps the first + /// and reports an AggregateException of both, and it has already put + /// the first one's message into a member of its own that this assembly + /// cannot set. Redacting by assignment therefore printed the key and + /// the redaction of it side by side — "One or more errors occurred. + /// (the key) (the redaction)" — and hid nothing. A result built here + /// has no first exception and no such member, so the runner reports + /// what is set below and nothing else. RedactionTests holds this. + /// + /// + private static TestResult Redacted(TestResult result) + => Redacted(result, Harness.Resource); + + /// + /// The same, against a resource key given here rather than the + /// configured one, so the redaction can be tested. + /// + internal static TestResult Redacted( + TestResult result, string? resource) + => new() + { + DisplayName = RedactedOrNull(result.DisplayName, resource), + Outcome = result.Outcome, + Duration = result.Duration, + ExecutionId = result.ExecutionId, + ParentExecId = result.ParentExecId, + ResultFiles = result.ResultFiles, + LogOutput = RedactedOrNull(result.LogOutput, resource), + LogError = RedactedOrNull(result.LogError, resource), + DebugTrace = RedactedOrNull(result.DebugTrace, resource), + TestContextMessages = + RedactedOrNull(result.TestContextMessages, resource), + TestFailureException = RedactedFailure(result, resource), + }; + + /// + /// The failure to report, redacted where it names the resource key. + /// The original is kept where it does not, so a reader still sees the + /// exception the test actually threw. + /// + private static Exception? RedactedFailure( + TestResult result, string? resource) { - result.LogOutput = RedactedOrNull(result.LogOutput); - result.LogError = RedactedOrNull(result.LogError); - result.DebugTrace = RedactedOrNull(result.DebugTrace); - result.TestContextMessages = RedactedOrNull(result.TestContextMessages); var failure = result.TestFailureException; - if (failure is null || string.IsNullOrEmpty(Harness.Resource)) + if (failure is null || string.IsNullOrEmpty(resource)) { - return; + return failure; } var whole = failure.ToString(); - if (whole.Contains(Harness.Resource, StringComparison.Ordinal) == false) + if (whole.Contains(resource, StringComparison.Ordinal) == false) { - return; + return failure; } // The exception cannot be edited, so a new one carries the redacted // text. Its own stack trace would point here, so the original one is // kept in the message, where the line that failed can still be read. var text = Harness.Redacted( $"{InnermostMessage(failure)}\nWhere it failed, kept because " - + $"the message was redacted:\n{whole}"); - result.TestFailureException = - result.Outcome == UnitTestOutcome.Inconclusive - ? new AssertInconclusiveException(text) - : new AssertFailedException(text); + + $"the message was redacted:\n{whole}", + resource); + return result.Outcome == UnitTestOutcome.Inconclusive + ? new AssertInconclusiveException(text) + : new AssertFailedException(text); } private static string InnermostMessage(Exception failure) @@ -84,8 +127,8 @@ private static string InnermostMessage(Exception failure) return inner.Message; } - private static string? RedactedOrNull(string? text) - => text is null ? null : Harness.Redacted(text); + private static string? RedactedOrNull(string? text, string? resource) + => text is null ? null : Harness.Redacted(text, resource); } /// diff --git a/Browser51Did/Harness.cs b/Browser51Did/Harness.cs index c6e1cdc..4cb1632 100644 --- a/Browser51Did/Harness.cs +++ b/Browser51Did/Harness.cs @@ -593,18 +593,46 @@ private static string Truncate(string value) /// never be written down anywhere. /// public static string Redacted(string text) - => string.IsNullOrEmpty(Resource) + => Redacted(text, Resource); + + /// + /// The same, against a resource key given here rather than the + /// configured one. The configured one is read once when this class is + /// first touched, so a test cannot set it; it passes its own instead. + /// + internal static string Redacted(string text, string? resource) + => string.IsNullOrEmpty(resource) ? text - : text.Replace(Resource, "", StringComparison.Ordinal); + : text.Replace( + resource, "", StringComparison.Ordinal); - /// One line of the script either side of the match. + /// + /// One line of the script either side of the match, with the resource + /// key taken out of it. + /// + /// The body this slices is the cloud-rendered client script, which + /// carries the resource key in the addresses it calls back on, so a + /// slice of it can contain the key wherever the match happens to fall. + /// The redaction is done here rather than at the call site because the + /// caller cannot tell from the returned text whether it needs it. + /// + /// private static string Around(string body, int at) + => Around(body, at, Resource); + + /// + /// The same, against a resource key given here rather than the + /// configured one, so the redaction can be tested. + /// + internal static string Around(string body, int at, string? resource) { var from = Math.Max(0, at - 40); var to = Math.Min(body.Length, at + 120); - return body.Substring(from, to - from) - .Replace("\r", " ", StringComparison.Ordinal) - .Replace("\n", " ", StringComparison.Ordinal); + return Redacted( + body.Substring(from, to - from) + .Replace("\r", " ", StringComparison.Ordinal) + .Replace("\n", " ", StringComparison.Ordinal), + resource); } #region Browsers diff --git a/Browser51Did/README.md b/Browser51Did/README.md index c4b87a2..e76967e 100644 --- a/Browser51Did/README.md +++ b/Browser51Did/README.md @@ -5,6 +5,11 @@ that a 51Did is created only after every other piece of data is complete and only from an answer the visitor actually gave, with the 51Degrees Preference Management Platform (PMP) and the client script on a publisher's page. +Six more, in +[RedactionTests.cs](RedactionTests.cs), prove that the resource key cannot +reach a test report. They need no cloud, no demo and no browser, so they run +whenever the category does. + They drive a **demo**, a small web app that serves every page the tests load. The dotnet demo is `Examples/Cloud/pmp-web` in [device-detection-dotnet-examples](https://github.com/51Degrees/device-detection-dotnet-examples). @@ -91,8 +96,9 @@ env CLOUD_ROOT_URL="http://localhost:5050/" \ digit, which is also why the name continuous integration sets starts with an underscore. In PowerShell write `${env:51DEGREES_RESOURCE_KEY} = '...'`. -With the endpoint or both resource key names unset every test reports -inconclusive with the reason. A resource key that cannot create reports inconclusive with the +With the endpoint or both resource key names unset every test that needs a +browser reports inconclusive with the reason, and the six redaction tests +still run. A resource key that cannot create reports inconclusive with the cloud's own refusal, and so does a cloud that will not hold a shared choice. The shared choice needs the browser to keep the cloud's `Secure` `SameSite=None` cookie, which both browsers did from plain HTTP on diff --git a/Browser51Did/RedactionTests.cs b/Browser51Did/RedactionTests.cs new file mode 100644 index 0000000..9464273 --- /dev/null +++ b/Browser51Did/RedactionTests.cs @@ -0,0 +1,237 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Browser51Did; + +/// +/// That a resource key cannot reach a test report. These tests need no +/// cloud, no demo and no browser, because what they check is the suite's +/// own handling of a result, so they run wherever the category runs. +/// +/// They exist because the redaction was once done by editing the result +/// MSTest had already built. That looked right and was not: assigning +/// a second time does not +/// replace the first exception, it aggregates with it, and the first one's +/// message has already been copied into a member of MSTest's own that this +/// assembly cannot set. The runner therefore reported the key and the +/// redaction of it side by side. The fix is to report a result built here +/// instead, which has neither. These tests fail against the edit-in-place +/// version and pass against the copy. +/// +/// +[TestClass, TestCategory("Browser51Did")] +public class RedactionTests +{ + /// + /// A resource key that is not one, long enough that it cannot appear + /// in a message by accident. + /// + private const string Key = "NOTAREALRESOURCEKEY0123456789"; + + /// + /// Text of the shape a failure here really takes: the client script's + /// address, which names the resource key. + /// + private const string Mentions = + "the script at https://cloud.example/" + Key + ".js was wrong"; + + /// + /// A result with the key in every piece of text it holds, standing in + /// for the one MSTest hands the attribute after a test has failed. + /// Every piece is set by reflection rather than by name, so a member + /// added by a later MSTest is covered without this test being changed. + /// + private static TestResult ResultMentioningTheKeyThroughout() + { + var result = new TestResult + { + Outcome = UnitTestOutcome.Failed, + TestFailureException = new AssertFailedException(Mentions), + }; + foreach (var field in TextFields()) + { + field.SetValue(result, Mentions); + } + return result; + } + + private static IEnumerable TextFields() + { + foreach (var field in typeof(TestResult).GetFields( + BindingFlags.Instance + | BindingFlags.Public + | BindingFlags.NonPublic)) + { + if (field.FieldType == typeof(string)) + { + yield return field; + } + } + } + + /// + /// The one that would have caught the bug. Nothing the runner can read + /// off the result may carry the key, including the members the public + /// interface does not name. + /// + [TestMethod] + public void NoTextOnTheReportedResultCarriesTheKey() + { + var reported = Browser51DidTestAttribute.Redacted( + ResultMentioningTheKeyThroughout(), Key); + foreach (var field in TextFields()) + { + var text = (string?)field.GetValue(reported); + Assert.IsFalse( + text != null + && text.Contains(Key, StringComparison.Ordinal), + $"TestResult.{field.Name} still carries the resource key " + + "after redaction, so a failing test would print it into " + + "a public build log. Anything the result holds has to be " + + "set from the redacted text or left unset."); + } + Assert.IsFalse( + reported.TestFailureException!.ToString() + .Contains(Key, StringComparison.Ordinal), + "the failure the runner reports still carries the resource " + + "key."); + } + + /// + /// Why the test above cannot be satisfied by editing the result in + /// place: MSTest holds the failure text somewhere this assembly cannot + /// name, so there is more to clear than the four public ones. If this + /// ever fails, MSTest has stopped keeping that copy and the copying in + /// can be reconsidered. + /// + [TestMethod] + public void MSTestKeepsTextBesideTheMembersThisSuiteCanSet() + { + var named = new[] + { + nameof(TestResult.DisplayName), + nameof(TestResult.LogOutput), + nameof(TestResult.LogError), + nameof(TestResult.DebugTrace), + nameof(TestResult.TestContextMessages), + }; + var unnamed = 0; + foreach (var field in TextFields()) + { + if (Array.Exists( + named, + name => field.Name.Contains(name, StringComparison.Ordinal)) + == false) + { + unnamed++; + } + } + Assert.IsTrue( + unnamed > 0, + "TestResult no longer holds text outside the members this " + + "suite sets by name, so the reason for copying the result " + + "rather than editing it has gone."); + } + + /// + /// The redaction has to leave the failure readable, or a build log says + /// only that something went wrong. + /// + [TestMethod] + public void TheRedactedFailureStillSaysWhatWentWrong() + { + var reported = Browser51DidTestAttribute.Redacted( + ResultMentioningTheKeyThroughout(), Key); + var message = reported.TestFailureException!.ToString(); + StringAssert.Contains( + message, + "the script at https://cloud.example/.js", + "the address that failed must still be readable with the key " + + "taken out of it."); + } + + /// + /// A skip is a result too, and its reason reaches the same log. + /// + [TestMethod] + public void AnInconclusiveResultStaysInconclusiveAndIsRedacted() + { + var skipped = new TestResult + { + Outcome = UnitTestOutcome.Inconclusive, + TestFailureException = new AssertInconclusiveException(Mentions), + }; + var reported = Browser51DidTestAttribute.Redacted(skipped, Key); + Assert.AreEqual( + UnitTestOutcome.Inconclusive, + reported.Outcome, + "a skip must still report as a skip."); + Assert.IsInstanceOfType( + reported.TestFailureException, + "a skip reported through a failure exception is counted as a " + + "failure."); + Assert.IsFalse( + reported.TestFailureException!.ToString() + .Contains(Key, StringComparison.Ordinal), + "the skip reason still carries the resource key."); + } + + /// + /// The copy must not quietly drop what the runner uses to place a + /// result, which is what makes a data row's result its own. + /// + [TestMethod] + public void TheCopyKeepsWhatIdentifiesTheResult() + { + var original = new TestResult + { + DisplayName = "a name with no key in it", + Outcome = UnitTestOutcome.Passed, + Duration = TimeSpan.FromSeconds(3), + ExecutionId = Guid.NewGuid(), + ParentExecId = Guid.NewGuid(), + ResultFiles = new List { "screenshot.png" }, + }; + var reported = Browser51DidTestAttribute.Redacted(original, Key); + Assert.AreEqual(original.DisplayName, reported.DisplayName); + Assert.AreEqual(original.Outcome, reported.Outcome); + Assert.AreEqual(original.Duration, reported.Duration); + Assert.AreEqual(original.ExecutionId, reported.ExecutionId); + Assert.AreEqual(original.ParentExecId, reported.ParentExecId); + CollectionAssert.AreEqual( + (List)original.ResultFiles!, + (List?)reported.ResultFiles, + "a result file the test left behind must still be reported."); + } + + /// + /// The guard prints a slice of the cloud-rendered client script to show + /// which template it was built from. That script calls back on + /// addresses naming the resource key, so a slice of it can contain the + /// key wherever the marker happens to fall. + /// + [TestMethod] + public void TheGuardsEvidenceSliceIsRedacted() + { + var marker = "fod.complete"; + var body = + "var url = 'https://cloud.example/" + Key + ".json';\n" + + "// " + marker + " is what says the script finished\n" + + "function done() { " + marker + "(); }"; + var at = body.IndexOf(marker, StringComparison.Ordinal); + var evidence = Harness.Around(body, at, Key); + StringAssert.Contains( + evidence, + marker, + "the slice must still show the marker it was taken around."); + Assert.IsFalse( + evidence.Contains(Key, StringComparison.Ordinal), + "the guard's evidence still carries the resource key. It is " + + "written to the console from ClassInitialize, outside " + + "Browser51DidTestAttribute, so nothing else takes it out."); + } +}