Run the Selenium contract tests against the on-premise web example, and fail the job when they fail

ci/run-integration-tests.ps1

@@ -36,37 +36,110 @@ if ($env:GITHUB_JOB -eq "Test") {
 $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
 
 ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys
-$status = $LASTEXITCODE
+$failures = @()
+if ($LASTEXITCODE -ne 0) {
+    $failures += "the Python integration tests exited with code $LASTEXITCODE"
+}
+
+# The shared Selenium contract tests are run against the cloud example and the
+# on-premise example in turn. Each result is also written as a trx file next
+# to the other integration results, so it is published with them.
+$resultsDir = (New-Item -ItemType Directory -Force -Path "$RepoName/test-results/integration").FullName
+
+# Get the shared contract tests.
+if (-not (Test-Path selenium-api-tests)) {
+    git clone --depth 1 https://github.com/51Degrees/selenium-api-tests.git
+    if ($LASTEXITCODE -ne 0) { throw "failed to clone selenium-api-tests" }
+}
 
-Write-Host 'Running Selenium tests...'
+# Settings the suite reads whichever example it drives. CLOUD_ROOT_URL is read
+# when the suite starts, so it is needed for the on-premise run too.
+$env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/"
+$env:PAID_RESOURCE_KEY = $Keys.TestResourceKey
+$env:EXAMPLE_LANG = 'python'
+
+# The examples run from their own virtual environment, created once here so a
+# running example never holds a file the second setup would need to replace.
+$examplesDir = (Resolve-Path "$PSScriptRoot/../fiftyone_devicedetection_examples").Path
+$py = Join-Path $examplesDir ($IsWindows ? ".venv/Scripts/python.exe" : ".venv/bin/python")
+Push-Location $examplesDir
 try {
-    # Start this repo's cloud example, pointed at the live cloud.
-    Push-Location "$PSScriptRoot/../fiftyone_devicedetection_examples"
+    python3 -m venv .venv
+    if ($LASTEXITCODE -ne 0) { throw "failed to create the examples virtual environment" }
+    & $py -m pip install -e .
+    if ($LASTEXITCODE -ne 0) { throw "failed to install the examples package" }
+} finally { Pop-Location }
+
+# Starts one of this repository's web examples on the given port, waits for it
+# to answer, runs the contract tests against it and stops it again. Returns a
+# description of the failure, or nothing when every test passed.
+function Test-Example([string]$Name, [string]$Module, [int]$Port) {
+    Write-Host "Running Selenium tests against the $Name example on port $Port..."
+    $example = $null
     try {
-        python3 -m venv .venv
-        $py = $IsWindows ? ".venv/Scripts/python.exe" : ".venv/bin/python"
-        & $py -m pip install -e .
-        $env:PORT = 8097
-        $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
-        $env:cloud_endpoint = "https://cloud.51degrees.com/api/v4/"
-        $example = & $py -m fiftyone_devicedetection_examples.cloud.gettingstarted_web 2>&1 &
-    } finally { Pop-Location }
-
-    # Get the shared contract tests.
-    if (-not (Test-Path selenium-api-tests)) {
-        git clone --depth 1 https://github.com/51Degrees/selenium-api-tests.git
+        Push-Location $examplesDir
+        try {
+            $env:PORT = $Port
+            $example = & $py -m $Module 2>&1 &
+        } finally { Pop-Location }
+
+        # Wait for the example to come up, or to stop.
+        $url = "http://localhost:$Port"
+        $deadline = (Get-Date).AddSeconds(180)
+        $ready = $false
+        while (-not $ready -and (Get-Date) -lt $deadline -and $example.State -eq 'Running') {
+            try {
+                Invoke-WebRequest -Uri $url -TimeoutSec 10 -SkipHttpErrorCheck | Out-Null
+                $ready = $true
+            } catch {
+                Start-Sleep -Seconds 2
+            }
+        }
+        if (-not $ready) {
+            return "the $Name example did not answer on $url (job state $($example.State))"
+        }
+
+        $env:EXAMPLE_URL = $url
+        # Output goes to the host, so that only a failure is returned.
+        dotnet test selenium-api-tests -c Release --filter TestCategory=Contract `
+            --results-directory $resultsDir --logger "trx;LogFileName=selenium-$Name.trx" `
+            --logger "console;verbosity=normal" | Out-Host
+        if ($LASTEXITCODE -ne 0) {
+            return "the Selenium contract tests against the $Name example exited with code $LASTEXITCODE"
+        }
+    } finally {
+        if ($example) {
+            Write-Host ">>> $Name example output >>>"
+            Receive-Job $example | Out-Host
+            Write-Host "<<< $Name example output <<<"
+            Remove-Job -Force $example
+        }
     }
-    # Wait for the example to come up.
-    curl -sS -o /dev/null --retry 5 --retry-connrefused "http://localhost:$env:PORT"
-
-    $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/"
-    $env:PAID_RESOURCE_KEY = $Keys.TestResourceKey
-    $env:EXAMPLE_URL = "http://localhost:$env:PORT"
-    $env:EXAMPLE_LANG = 'python'
-    dotnet test selenium-api-tests -c Release --filter TestCategory=Contract
-} catch {
-    if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' }
-    throw
-} finally {
-    if ($example) { Remove-Job -Force $example }
+}
+
+# The cloud example, pointed at the live cloud.
+$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
+$env:cloud_endpoint = "https://cloud.51degrees.com/api/v4/"
+$failures += Test-Example -Name 'cloud' `
+    -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097
+
+# The on-premise example. The contract tests need the device type and the
+# JavaScript properties, which the Lite data file does not have, so it runs
+# against the TAC data file, which is only fetched when a licence is given.
+$tac = "$PWD/assets/TAC-HashV41.hash"
+if (Test-Path $tac) {
+    ${env:51DEGREES_DD_PATH} = (Resolve-Path $tac).Path
+    $failures += Test-Example -Name 'onpremise' `
+        -Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098
+    Remove-Item Env:51DEGREES_DD_PATH
+} else {
+    Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example."
+}
+
+# A failure anywhere above must fail the job. This script is dot-sourced by
+# common-ci, so an exit code is lost once a later step runs a command of its
+# own, and only an error that stops the script is seen by the job.
+$failures = @($failures | Where-Object { $_ })
+if ($failures.Count -gt 0) {
+    throw "Integration tests failed: $($failures -join '; ')"
 }

fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py

@@ -46,6 +46,11 @@ class ExampleUtils:
 
     ENDPOINT_ENV_VAR = "cloud_endpoint"
 
+    # The environment variable an on-premise example reads the path of its
+    # device detection data file from, when one is given, in place of the
+    # file named in its configuration.
+    DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH"
+
     @staticmethod
     def get_resource_key_from_config(config):
         key = ""
@@ -163,6 +168,10 @@ def get_missing_resource_key_message():
                 "read). Create a resource key for free at "
                 "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=device-detection-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=resource-key-required")
 
+    @staticmethod
+    def get_data_file_path():
+        return ExampleUtils.__get_env_variable(ExampleUtils.DATA_FILE_ENV_VAR)
+
     @staticmethod
     def get_cloud_endpoint():
         return ExampleUtils.__get_env_variable(ExampleUtils.ENDPOINT_ENV_VAR)

fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py

@@ -99,8 +99,10 @@ def build(self, config, logger):
         return self
 
     def run(self):
-
-        GettingStartedWeb.app.run(port=5001)
+
+        # The port can be set with the PORT environment variable, as it can
+        # for the cloud example, so the example can run next to other services.
+        GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001)))
 
     # First we make a JSON route that will be called from the client side and will return
     # a JSON encoded property database using any additional evidence provided by the client 
@@ -125,6 +127,36 @@ def jsonroute():
 
         return json.dumps(flowdata.jsonbundler.json)
 
+    # Next we serve the client-side JavaScript from its own route. The other
+    # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js'
+    # for this, so the page can reference the script by that name. There is no
+    # such integration for Flask, so the example wires up the route itself.
+
+    @staticmethod
+    @app.route('/51Degrees.core.js')
+    def core_js():
+
+        # Create the flowdata object for the JavaScript route
+        flowdata = GettingStartedWeb.pipeline.create_flowdata()
+
+        # Add any information from the request (headers, cookies and additional
+        # client side provided information). Query parameters are included, so
+        # the per-request 'fod-js-enable-cookies' override is picked up here and
+        # applied by the JavaScriptBuilder engine.
+
+        flowdata.evidence.add_from_dict(webevidence(request))
+
+        # Process the flowdata
+
+        flowdata.process()
+
+        # Return the JavaScript from the JavaScriptBuilder engine
+
+        response = make_response(flowdata.javascriptbuilder.javascript)
+        response.headers["Content-Type"] = "application/x-javascript"
+
+        return response
+
     # In the main route we dynamically update the screen's device property display
     # using the above JSON route
 
@@ -182,6 +214,19 @@ def build_config():
         configFile = Path(__file__).resolve().parent.joinpath("config.json").read_text()
         config = json.loads(configFile)
 
+        # A data file named in the 51DEGREES_DD_PATH environment variable takes
+        # the place of the one in config.json. This lets the example run
+        # against a paid data file without editing the configuration.
+        envDataFile = ExampleUtils.get_data_file_path()
+        if envDataFile:
+            envDataFile = os.path.abspath(envDataFile)
+            if not os.path.exists(envDataFile):
+                raise Exception("The device detection data file " +
+                    f"'{envDataFile}' named in the environment variable " +
+                    f"'{ExampleUtils.DATA_FILE_ENV_VAR}' does not exist.")
+            ExampleUtils.set_data_file_in_config(config, envDataFile)
+            return config
+
         dataFile = ExampleUtils.get_data_file_from_config(config)
         foundDataFile = False
         if not dataFile:
@@ -203,7 +248,8 @@ def build_config():
                 f"'{dataFile}'. If using the lite file, then make sure the " +
                 "device-detection-data submodule has been updated by running " +
                 "`git submodule update --recursive`. Otherwise, ensure that the filename " +
-                "is correct in config.json.")
+                "is correct in config.json, or set the environment variable " +
+                f"'{ExampleUtils.DATA_FILE_ENV_VAR}' to the path of the data file.")
 
         return config
 

fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/config.json

@@ -11,6 +11,10 @@
                     "licence_keys": ""
                 }
             },
+            {
+                "elementName": "SequenceElement",
+                "elementPath": "fiftyone_pipeline_core.sequenceelement"
+            },
             {
                 "elementName": "JSONBundlerElement",
                 "elementPath": "fiftyone_pipeline_core.jsonbundler"

fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html

@@ -141,6 +141,9 @@ <h3 class="c-eg-page__heading">Device data</h3>
                 <tr class="c-eg-table__row c-eg-table__row--alt"><td class="c-eg-table__cell c-eg-table__cell--key">Browser Version:</td><td class="c-eg-table__cell">{{ utils.get_human_readable(data.device, "browserversion") }}</td></tr>
                 <tr class="c-eg-table__row"><td class="c-eg-table__cell c-eg-table__cell--key">Screen width (pixels):</td><td class="c-eg-table__cell">{{ utils.get_human_readable(data.device, "screenpixelswidth") }}</td></tr>
                 <tr class="c-eg-table__row c-eg-table__row--alt"><td class="c-eg-table__cell c-eg-table__cell--key">Screen height (pixels):</td><td class="c-eg-table__cell">{{ utils.get_human_readable(data.device, "screenpixelsheight") }}</td></tr>
+                {# The device id identifies the matched hardware, platform and browser profiles,
+                   so it is the compact form of everything else in this table. #}
+                <tr class="c-eg-table__row"><td class="c-eg-table__cell c-eg-table__cell--key">Device Id:</td><td class="c-eg-table__cell">{{ utils.get_human_readable(data.device, "deviceid") }}</td></tr>
             </tbody>
         </table>
 
@@ -165,18 +168,13 @@ <h3 class="c-eg-page__heading">Client-side evidence and Apple models</h3>
 </div>
 
 {#
-    This script is constructed by the fiftyone_pipeline_core package.
-    It adds a JavaScript include for 51Degrees.core.js.
-    The 51Degrees pipeline will dynamically generate JavaScript, which includes a
-    JSON representation of the contents of flow data, i.e. the results from device detection.
-    That script raises the 'complete' event with the refined flow data. The shared
-    examples.js helper subscribes to it and appends a results table into #content.
+    51Degrees.core.js is built by the fiftyone_pipeline_core package and served by the
+    matching route in app.py. It includes a JSON representation of the contents of flow
+    data, i.e. the results from device detection, and raises the 'complete' event with the
+    refined flow data. The shared examples.js helper subscribes to it and appends a results
+    table into #content.
 #}
-{% autoescape false %}
-<script>
-{{ data.javascriptbuilder.javascript }}
-</script>
-{% endautoescape %}
+<script src="/51Degrees.core.js"></script>
 
 <script src="{{ url_for('static', filename='js/examples.min.js') }}"></script>
 <script>

fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py

@@ -21,7 +21,11 @@
 # *********************************************************************
 
 import flask_unittest
+import os
+import re
+import tempfile
 import unittest
+from unittest import mock
 from fiftyone_pipeline_core.logger import Logger
 from fiftyone_devicedetection_examples.example_utils import ExampleUtils
 from fiftyone_devicedetection_examples.onpremise.gettingstarted_web.app import GettingStartedWeb
@@ -38,3 +42,51 @@ class OnPremiseGettingStartedWebTests(flask_unittest.ClientTestCase):
     def test_onpremise_getting_started_web(self, client):
         response = client.get('/')
         self.assertEqual(200, response.status_code)
+
+    # The page references the client-side script by the '/51Degrees.core.js' name used
+    # by the web integrations in the other Pipeline APIs, so check that the route
+    # returns the bundle rather than, for example, falling through to the page.
+    def test_onpremise_getting_started_web_core_js(self, client):
+        response = client.get('/51Degrees.core.js')
+        self.assertEqual(200, response.status_code)
+        self.assertEqual("application/x-javascript", response.headers["Content-Type"])
+        self.assertIn(b"fiftyoneDegreesManager", response.data)
+
+    # The script takes its sequence number from the SequenceElement. Without that
+    # element in config.json the script is rendered as 'var sequence=;', which
+    # does not parse, so the browser never defines 'fod'.
+    def test_onpremise_getting_started_web_core_js_has_sequence(self, client):
+        response = client.get('/51Degrees.core.js')
+        self.assertIsNone(
+            re.search(rb"sequence\s*=\s*;", response.data),
+            "the script has no sequence number, so it will not parse")
+
+    # The page must load the script from that route rather than inline it.
+    def test_onpremise_getting_started_web_references_core_js(self, client):
+        response = client.get('/')
+        self.assertIn(b'<script src="/51Degrees.core.js"></script>', response.data)
+
+
+class OnPremiseGettingStartedWebConfigTests(unittest.TestCase):
+
+    # A data file named in the environment replaces the one in config.json.
+    def test_data_file_from_environment(self):
+        with tempfile.NamedTemporaryFile(suffix=".hash", delete=False) as file:
+            path = file.name
+        try:
+            with mock.patch.dict(os.environ, {ExampleUtils.DATA_FILE_ENV_VAR: path}):
+                config = GettingStartedWeb.build_config()
+            self.assertEqual(
+                os.path.abspath(path),
+                ExampleUtils.get_data_file_from_config(config))
+        finally:
+            os.remove(path)
+
+    # A data file named in the environment that does not exist is reported by
+    # name, rather than the example silently falling back to config.json.
+    def test_missing_data_file_from_environment(self):
+        missing = os.path.join(tempfile.gettempdir(), "no-such-51degrees-file.hash")
+        with mock.patch.dict(os.environ, {ExampleUtils.DATA_FILE_ENV_VAR: missing}):
+            with self.assertRaises(Exception) as context:
+                GettingStartedWeb.build_config()
+        self.assertIn(ExampleUtils.DATA_FILE_ENV_VAR, str(context.exception))