From 633f84cd32bca1210a45b47cfa5d74aff9039b58 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:11:34 +0100 Subject: [PATCH 1/3] FIX: Let the browser test result reach the job, and allow a suite branch A failing dotnet test returns non-zero, and PowerShell carries on, so the Selenium step could fail every case and the job was still reported as a success. On the ARM64 Linux job it did exactly that, with nine failures in the log and a green tick on the run. The exit code is now checked and the step stops. SELENIUM_TESTS_REF picks a branch of the shared browser suite, defaulting to main, so a change there can be tried here before it is merged. --- ci/run-integration-tests.ps1 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 3c269def7..6818225d4 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -88,9 +88,12 @@ try { $example = java -jar $jar 2>&1 & } finally { Pop-Location } - # Get the shared contract tests. + # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of + # the suite, so a change there can be tried here before it is merged. + $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { - git clone --depth 1 https://github.com/51Degrees/selenium-api-tests.git + Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" + git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git } # Wait for the example to come up. curl -sS -o /dev/null --retry 5 --retry-connrefused "http://localhost:$env:PORT" @@ -100,6 +103,13 @@ try { $env:EXAMPLE_URL = "http://localhost:$env:PORT" $env:EXAMPLE_LANG = 'java' dotnet test selenium-api-tests -c Release --filter TestCategory=Contract + # PowerShell does not stop for a native command that returns non-zero, so + # without this the browser tests could fail every case and the job would + # still be reported as a success. + if ($LASTEXITCODE -ne 0) { + throw "The Selenium contract tests failed, returning $LASTEXITCODE. " + + "Their output is above." + } } catch { if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } throw From cf39ce8ea0ef5b9e27aa47b0537d2f780e1b09fe Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:12:04 +0100 Subject: [PATCH 2/3] CI: Proof run only, reverted next commit Points the browser suite at the branch carrying the ARM64 fix and trims the matrix to the ARM64 Linux job, so one dispatched run shows whether the browser tests really run there. --- ci/options.json | 97 +----------------------------------- ci/run-integration-tests.ps1 | 3 ++ 2 files changed, 4 insertions(+), 96 deletions(-) diff --git a/ci/options.json b/ci/options.json index 960779b82..796ec8516 100644 --- a/ci/options.json +++ b/ci/options.json @@ -1,104 +1,9 @@ [ - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_X64", - "RunPerformance": true - }, - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, - "PackageRequirement" : true - }, - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_X64", - "RunPerformance": true - }, { "Image": "ubuntu-22.04-arm", "Name": "Ubuntu_ARM_Java_17", "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, - "PackageRequirement" : true - }, - { - "Image": "ubuntu-22.04", - "Name": "Ubuntu_Java_8", - "JavaSDKEnvVar": "JAVA_HOME_8_X64", - "RunPerformance": true - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_X64", - "RunPerformance": true, - "PackageRequirement" : false - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, - "PackageRequirement" : true - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_X64", - "RunPerformance": true - }, - { - "Image": "windows-latest", - "Name": "Windows_Java_8", - "JavaSDKEnvVar": "JAVA_HOME_8_X64", - "RunPerformance": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_X64", - "RunPerformance": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": true, + "RunPerformance": false, "PackageRequirement": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_X64", - "RunPerformance": true - }, - { - "Image": "macos-15-intel", - "Name": "MacOS_X64_Java_8", - "JavaSDKEnvVar": "JAVA_HOME_8_X64", - "RunPerformance": true - }, - { - "Image": "macos-15", - "Name": "MacOS_ARM_Java_21", - "JavaSDKEnvVar": "JAVA_HOME_21_arm64", - "RunPerformance": true - }, - { - "Image": "macos-15", - "Name": "MacOS_ARM_Java_17", - "JavaSDKEnvVar": "JAVA_HOME_17_arm64", - "RunPerformance": true, - "PackageRequirement": true - }, - { - "Image": "macos-15", - "Name": "MacOS_ARM_Java_11", - "JavaSDKEnvVar": "JAVA_HOME_11_arm64", - "RunPerformance": true } ] diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 6818225d4..6372a8610 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -90,6 +90,9 @@ try { # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of # the suite, so a change there can be tried here before it is merged. + # PROOF RUN ONLY, reverted in the next commit: point the suite at the + # branch that carries the ARM64 fix. + $env:SELENIUM_TESTS_REF = 'fix/arm-linux-browser-drivers' $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" From 4c541536f0d5a981d7328666b77cd72da6192928 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 17 Sep 2026 09:17:57 +0100 Subject: [PATCH 3/3] Revert "CI: Proof run only, reverted next commit" This reverts the temporary pin and matrix trim used for run 35198427368, which proved the browser tests really run on the ARM64 Linux job. The matrix is back to the full list and the suite is read from main again. --- ci/options.json | 97 +++++++++++++++++++++++++++++++++++- ci/run-integration-tests.ps1 | 3 -- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/ci/options.json b/ci/options.json index 796ec8516..960779b82 100644 --- a/ci/options.json +++ b/ci/options.json @@ -1,9 +1,104 @@ [ + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_X64", + "RunPerformance": true + }, + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_X64", + "RunPerformance": true, + "PackageRequirement" : true + }, + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_X64", + "RunPerformance": true + }, { "Image": "ubuntu-22.04-arm", "Name": "Ubuntu_ARM_Java_17", "JavaSDKEnvVar": "JAVA_HOME_17_X64", - "RunPerformance": false, + "RunPerformance": true, + "PackageRequirement" : true + }, + { + "Image": "ubuntu-22.04", + "Name": "Ubuntu_Java_8", + "JavaSDKEnvVar": "JAVA_HOME_8_X64", + "RunPerformance": true + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_X64", + "RunPerformance": true, + "PackageRequirement" : false + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_X64", + "RunPerformance": true, + "PackageRequirement" : true + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_X64", + "RunPerformance": true + }, + { + "Image": "windows-latest", + "Name": "Windows_Java_8", + "JavaSDKEnvVar": "JAVA_HOME_8_X64", + "RunPerformance": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_X64", + "RunPerformance": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_X64", + "RunPerformance": true, "PackageRequirement": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_X64", + "RunPerformance": true + }, + { + "Image": "macos-15-intel", + "Name": "MacOS_X64_Java_8", + "JavaSDKEnvVar": "JAVA_HOME_8_X64", + "RunPerformance": true + }, + { + "Image": "macos-15", + "Name": "MacOS_ARM_Java_21", + "JavaSDKEnvVar": "JAVA_HOME_21_arm64", + "RunPerformance": true + }, + { + "Image": "macos-15", + "Name": "MacOS_ARM_Java_17", + "JavaSDKEnvVar": "JAVA_HOME_17_arm64", + "RunPerformance": true, + "PackageRequirement": true + }, + { + "Image": "macos-15", + "Name": "MacOS_ARM_Java_11", + "JavaSDKEnvVar": "JAVA_HOME_11_arm64", + "RunPerformance": true } ] diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 6372a8610..6818225d4 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -90,9 +90,6 @@ try { # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of # the suite, so a change there can be tried here before it is merged. - # PROOF RUN ONLY, reverted in the next commit: point the suite at the - # branch that carries the ARM64 fix. - $env:SELENIUM_TESTS_REF = 'fix/arm-linux-browser-drivers' $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { Write-Host "Cloning the Selenium contract tests from '$seleniumRef'"