From 106859e34828968ada12cdea7498c5e6e7dea548 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 16 Sep 2026 22:28:44 +0100 Subject: [PATCH 1/6] TEST: Run the Selenium contract tests against the on-premise web example too The integration tests already ran the shared contract tests against the cloud web example. They now also start the on-premise web example on port 8098 with the TAC data file, because the Lite file lacks DeviceType and the JavaScript properties the tests need. A failing Selenium run now fails the job. Before, the exit code of dotnet test was overwritten by later steps, so a run where all nine tests failed on the Arm64 Linux runner still reported success. That runner is now skipped with a message, because the Selenium Manager binary the suite ships for Linux is x64 only. The wait for the example uses Invoke-WebRequest, because curl on Windows could not write to /dev/null and returned exit code 23. The on-premise example must first be able to take its port and data file from the environment, a change in device-detection-java-examples that has to merge before this one. --- ci/run-integration-tests.ps1 | 132 +++++++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 22 deletions(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 3c269def7..81b68a227 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -75,34 +75,122 @@ finally { $status = $LASTEXITCODE -Write-Host 'Running Selenium tests...' -try { - # Build and start the cloud example (already pinned to the dev version above). - Push-Location "device-detection-java-examples" +# Runs the shared Selenium contract tests (category Contract in +# selenium-api-tests) against one web example. The example is built, started +# in the background on its own port and left to serve until the tests finish. +# Any failure is added to $Failures rather than thrown, so that one example +# failing does not stop the other from being tested. +function Invoke-ContractTests { + param( + [Parameter(Mandatory)][string]$Label, + [Parameter(Mandatory)][string]$Module, + [Parameter(Mandatory)][int]$Port, + [Parameter(Mandatory)][System.Collections.Generic.List[string]]$Failures, + # Environment variables the example needs, removed again afterwards. + [hashtable]$ExampleEnv = @{} + ) + Write-Host "Running Selenium contract tests against the $Label example on port $Port" + $example = $null + $failure = $null + $stdout = Join-Path (Get-Location).Path "example-$Port.out.log" + $stderr = Join-Path (Get-Location).Path "example-$Port.err.log" try { - mvn -B --no-transfer-progress -pl web/getting-started.cloud -am package -DskipTests - $jar = (Get-ChildItem web/getting-started.cloud/target/*-jar-with-dependencies.jar | Select-Object -First 1).FullName - $env:PORT = 8099 - $env:TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" - $env:TestResourceKey = $Keys.TestResourceKey - $example = java -jar $jar 2>&1 & - } finally { Pop-Location } + # Build and start the example (already pinned to the dev version above). + # The example finds its web content relative to the examples repository + # root, so the process is started from there. + Push-Location "device-detection-java-examples" + try { + mvn -B --no-transfer-progress -pl $Module -am package -DskipTests | Out-Host + if ($LASTEXITCODE -ne 0) { + $failure = "the $Label example failed to build" + } else { + $jar = (Get-ChildItem "$Module/target/*-jar-with-dependencies.jar" | Select-Object -First 1).FullName + $env:PORT = $Port + foreach ($name in $ExampleEnv.Keys) { + Set-Item "env:$name" $ExampleEnv[$name] + } + $example = Start-Process java -ArgumentList '-jar', "`"$jar`"" ` + -WorkingDirectory (Get-Location).Path -NoNewWindow -PassThru ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr + } + } finally { Pop-Location } + + if (-not $failure) { + # Wait up to three minutes for the example to answer. + $up = $false + for ($i = 0; $i -lt 60 -and -not $up; $i++) { + try { + Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 "http://localhost:$Port" | Out-Null + $up = $true + } catch { + if ($example.HasExited) { break } + Start-Sleep -Seconds 3 + } + } + if (-not $up) { + $failure = "the $Label example did not answer on port $Port" + } + } + if (-not $failure) { + $env:EXAMPLE_URL = "http://localhost:$Port" + $env:EXAMPLE_LANG = 'java' + dotnet test selenium-api-tests -c Release --filter TestCategory=Contract | Out-Host + if ($LASTEXITCODE -ne 0) { + $failure = "the contract tests failed against the $Label example" + } + } + + if ($failure) { + $Failures.Add($failure) + if ($example) { + Write-Host ">>> $Label example output >>>" + Get-Content $stdout, $stderr -ErrorAction SilentlyContinue | Out-Host + Write-Host "<<< $Label example output <<<" + } + } + } finally { + if ($example -and -not $example.HasExited) { + Stop-Process -Id $example.Id -Force + } + Remove-Item env:PORT, env:EXAMPLE_URL -ErrorAction SilentlyContinue + foreach ($name in $ExampleEnv.Keys) { + Remove-Item "env:$name" -ErrorAction SilentlyContinue + } + } +} + +Write-Host 'Running Selenium tests...' +if ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture -eq 'Arm64') { + # The Selenium Manager binary that the suite ships for Linux is built for + # x64 only, so on an Arm64 Linux runner every test fails with + # "Exec format error" before a browser is started. + Write-Host 'Skipping Selenium tests, Selenium Manager has no Linux Arm64 build' +} else { # Get the shared contract tests. if (-not (Test-Path selenium-api-tests)) { git clone --depth 1 https://github.com/51Degrees/selenium-api-tests.git + if ($LASTEXITCODE -ne 0) { throw "Failed to clone selenium-api-tests" } } - # Wait for the example to come up. - curl -sS -o /dev/null --retry 5 --retry-connrefused "http://localhost:$env:PORT" - $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" $env:PAID_RESOURCE_KEY = $Keys.TestResourceKey - $env:EXAMPLE_URL = "http://localhost:$env:PORT" - $env:EXAMPLE_LANG = 'java' - dotnet test selenium-api-tests -c Release --filter TestCategory=Contract -} catch { - if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } - throw -} finally { - if ($example) { Remove-Job -Force $example } + + $seleniumFailures = [System.Collections.Generic.List[string]]::new() + + # The cloud example. + Invoke-ContractTests -Label 'cloud' -Module 'web/getting-started.cloud' -Port 8099 -Failures $seleniumFailures -ExampleEnv @{ + TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" + TestResourceKey = $Keys.TestResourceKey + } + + # The on-premise example, against the TAC data file copied into the + # examples repository above. The Lite data file has neither DeviceType nor + # the JavaScript properties the contract tests need. + Invoke-ContractTests -Label 'on-premise' -Module 'web/getting-started.onprem' -Port 8098 -Failures $seleniumFailures -ExampleEnv @{ + '51DEGREES_DD_PATH' = (Resolve-Path "device-detection-java-examples/device-detection-data/TAC-HashV41.hash").Path + } + + if ($seleniumFailures.Count -gt 0) { + throw "Selenium tests failed: $($seleniumFailures -join '; ')" + } } From 23ee2e4853276851b06d5db9cca97c7ed0acf7b5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 16 Sep 2026 22:38:18 +0100 Subject: [PATCH 2/6] TEST: Allow the Selenium failure list to start empty A mandatory parameter refuses an empty collection, so the first call failed before any test ran. --- ci/run-integration-tests.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 81b68a227..1129316ed 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -85,7 +85,8 @@ function Invoke-ContractTests { [Parameter(Mandatory)][string]$Label, [Parameter(Mandatory)][string]$Module, [Parameter(Mandatory)][int]$Port, - [Parameter(Mandatory)][System.Collections.Generic.List[string]]$Failures, + [Parameter(Mandatory)][AllowEmptyCollection()] + [System.Collections.Generic.List[string]]$Failures, # Environment variables the example needs, removed again afterwards. [hashtable]$ExampleEnv = @{} ) From 0ebdda95c679569ae35828c22b9cd9612452ba93 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 16 Sep 2026 22:40:08 +0100 Subject: [PATCH 3/6] TEST: Fail the job when the example tests fail The exit code of the examples' mvn clean test was stored in $status and never read, so failing example tests left the job green. The code is now kept and reported, together with any Selenium failure, once all the tests have run. --- ci/run-integration-tests.ps1 | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 1129316ed..6b19773d6 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -14,6 +14,7 @@ param( $RepoPath = [IO.Path]::Combine($pwd, $RepoName) $ExamplesRepoName = "$RepoName-examples" +$examplesStatus = 0 try { Write-Output "Cloning '$ExamplesRepoName'" @@ -47,6 +48,8 @@ try { Write-Output "Testing Examples" mvn clean test "-DTestResourceKey=$($Keys.TestResourceKey)" "-DSuperResourceKey=$($Keys.TestResourceKey)" "-DLicenseKey=$($Keys.DeviceDetection)" + # Checked after the Selenium tests, so a failure here does not stop them. + $examplesStatus = $LASTEXITCODE Write-Output "Copying test results". # Copy the test results into the test-results folder @@ -73,8 +76,6 @@ finally { } -$status = $LASTEXITCODE - # Runs the shared Selenium contract tests (category Contract in # selenium-api-tests) against one web example. The example is built, started # in the background on its own port and left to serve until the tests finish. @@ -161,6 +162,12 @@ function Invoke-ContractTests { } } +# Failures are collected so that every set of tests runs before the job fails. +$failures = [System.Collections.Generic.List[string]]::new() +if ($examplesStatus -ne 0) { + $failures.Add("the example tests failed with exit code $examplesStatus") +} + Write-Host 'Running Selenium tests...' if ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture -eq 'Arm64') { # The Selenium Manager binary that the suite ships for Linux is built for @@ -176,10 +183,8 @@ if ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::OSArchite $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" $env:PAID_RESOURCE_KEY = $Keys.TestResourceKey - $seleniumFailures = [System.Collections.Generic.List[string]]::new() - # The cloud example. - Invoke-ContractTests -Label 'cloud' -Module 'web/getting-started.cloud' -Port 8099 -Failures $seleniumFailures -ExampleEnv @{ + Invoke-ContractTests -Label 'cloud' -Module 'web/getting-started.cloud' -Port 8099 -Failures $failures -ExampleEnv @{ TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" TestResourceKey = $Keys.TestResourceKey } @@ -187,11 +192,11 @@ if ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::OSArchite # The on-premise example, against the TAC data file copied into the # examples repository above. The Lite data file has neither DeviceType nor # the JavaScript properties the contract tests need. - Invoke-ContractTests -Label 'on-premise' -Module 'web/getting-started.onprem' -Port 8098 -Failures $seleniumFailures -ExampleEnv @{ + Invoke-ContractTests -Label 'on-premise' -Module 'web/getting-started.onprem' -Port 8098 -Failures $failures -ExampleEnv @{ '51DEGREES_DD_PATH' = (Resolve-Path "device-detection-java-examples/device-detection-data/TAC-HashV41.hash").Path } +} - if ($seleniumFailures.Count -gt 0) { - throw "Selenium tests failed: $($seleniumFailures -join '; ')" - } +if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" } From 633f84cd32bca1210a45b47cfa5d74aff9039b58 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:11:34 +0100 Subject: [PATCH 4/6] FIX: Let the browser test result reach the job, and allow a suite branch A failing dotnet test returns non-zero, and PowerShell carries on, so the Selenium step could fail every case and the job was still reported as a success. On the ARM64 Linux job it did exactly that, with nine failures in the log and a green tick on the run. The exit code is now checked and the step stops. SELENIUM_TESTS_REF picks a branch of the shared browser suite, defaulting to main, so a change there can be tried here before it is merged. --- ci/run-integration-tests.ps1 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 3c269def7..6818225d4 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -88,9 +88,12 @@ try { $example = java -jar $jar 2>&1 & } finally { Pop-Location } - # Get the shared contract tests. + # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of + # the suite, so a change there can be tried here before it is merged. + $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { - git clone --depth 1 https://github.com/51Degrees/selenium-api-tests.git + Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" + git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git } # Wait for the example to come up. curl -sS -o /dev/null --retry 5 --retry-connrefused "http://localhost:$env:PORT" @@ -100,6 +103,13 @@ try { $env:EXAMPLE_URL = "http://localhost:$env:PORT" $env:EXAMPLE_LANG = 'java' dotnet test selenium-api-tests -c Release --filter TestCategory=Contract + # PowerShell does not stop for a native command that returns non-zero, so + # without this the browser tests could fail every case and the job would + # still be reported as a success. + if ($LASTEXITCODE -ne 0) { + throw "The Selenium contract tests failed, returning $LASTEXITCODE. " + + "Their output is above." + } } catch { if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } throw From cf39ce8ea0ef5b9e27aa47b0537d2f780e1b09fe Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:12:04 +0100 Subject: [PATCH 5/6] CI: Proof run only, reverted next commit Points the browser suite at the branch carrying the ARM64 fix and trims the matrix to the ARM64 Linux job, so one dispatched run shows whether the browser tests really run there. --- ci/options.json | 97 +----------------------------------- ci/run-integration-tests.ps1 | 3 ++ 2 files changed, 4 insertions(+), 96 deletions(-) diff --git a/ci/options.json b/ci/options.json index 960779b82..796ec8516 100644 --- a/ci/options.json +++ b/ci/options.json @@ -1,104 +1,9 @@ [ - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_X64", - "RunPerformance": true - }, - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, - "PackageRequirement" : true - }, - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_X64", - "RunPerformance": true - }, { "Image": "ubuntu-22.04-arm", "Name": "Ubuntu_ARM_Java_17", "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, - "PackageRequirement" : true - }, - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_8", - "JavaSDKEnvVar": "JAVA_HOME_8_X64", - "RunPerformance": true - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_X64", - "RunPerformance": true, - "PackageRequirement" : false - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, - "PackageRequirement" : true - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_X64", - "RunPerformance": true - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_8", - "JavaSDKEnvVar": "JAVA_HOME_8_X64", - "RunPerformance": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_X64", - "RunPerformance": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, + "RunPerformance": false, "PackageRequirement": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_X64", - "RunPerformance": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_8", - "JavaSDKEnvVar": "JAVA_HOME_8_X64", - "RunPerformance": true - }, - { - "Image": "macos-15", - "Name": "MacOS_ARM_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_arm64", - "RunPerformance": true - }, - { - "Image": "macos-15", - "Name": "MacOS_ARM_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_arm64", - "RunPerformance": true, - "PackageRequirement": true - }, - { - "Image": "macos-15", - "Name": "MacOS_ARM_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_arm64", - "RunPerformance": true } ] diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 6818225d4..6372a8610 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -90,6 +90,9 @@ try { # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of # the suite, so a change there can be tried here before it is merged. + # PROOF RUN ONLY, reverted in the next commit: point the suite at the + # branch that carries the ARM64 fix. + $env:SELENIUM_TESTS_REF = 'fix/arm-linux-browser-drivers' $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" From 4c541536f0d5a981d7328666b77cd72da6192928 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:17:57 +0100 Subject: [PATCH 6/6] Revert "CI: Proof run only, reverted next commit" This reverts the temporary pin and matrix trim used for run 35198427368, which proved the browser tests really run on the ARM64 Linux job. The matrix is back to the full list and the suite is read from main again. --- ci/options.json | 97 +++++++++++++++++++++++++++++++++++- ci/run-integration-tests.ps1 | 3 -- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/ci/options.json b/ci/options.json index 796ec8516..960779b82 100644 --- a/ci/options.json +++ b/ci/options.json @@ -1,9 +1,104 @@ [ + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_X64", + "RunPerformance": true + }, + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_X64", + "RunPerformance": true, + "PackageRequirement" : true + }, + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_X64", + "RunPerformance": true + }, { "Image": "ubuntu-22.04-arm", "Name": "Ubuntu_ARM_Java_17", "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": false, + "RunPerformance": true, + "PackageRequirement" : true + }, + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_8", + "JavaSDKEnvVar": "JAVA_HOME_8_X64", + "RunPerformance": true + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_X64", + "RunPerformance": true, + "PackageRequirement" : false + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_X64", + "RunPerformance": true, + "PackageRequirement" : true + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_X64", + "RunPerformance": true + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_8", + "JavaSDKEnvVar": "JAVA_HOME_8_X64", + "RunPerformance": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_X64", + "RunPerformance": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_X64", + "RunPerformance": true, "PackageRequirement": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_X64", + "RunPerformance": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_8", + "JavaSDKEnvVar": "JAVA_HOME_8_X64", + "RunPerformance": true + }, + { + "Image": "macos-15", + "Name": "MacOS_ARM_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_arm64", + "RunPerformance": true + }, + { + "Image": "macos-15", + "Name": "MacOS_ARM_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_arm64", + "RunPerformance": true, + "PackageRequirement": true + }, + { + "Image": "macos-15", + "Name": "MacOS_ARM_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_arm64", + "RunPerformance": true } ] diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 6372a8610..6818225d4 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -90,9 +90,6 @@ try { # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of # the suite, so a change there can be tried here before it is merged. - # PROOF RUN ONLY, reverted in the next commit: point the suite at the - # branch that carries the ARM64 fix. - $env:SELENIUM_TESTS_REF = 'fix/arm-linux-browser-drivers' $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { Write-Host "Cloning the Selenium contract tests from '$seleniumRef'"