diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index 3c269def7..4a246614a 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -14,6 +14,7 @@ param( $RepoPath = [IO.Path]::Combine($pwd, $RepoName) $ExamplesRepoName = "$RepoName-examples" +$examplesStatus = 0 try { Write-Output "Cloning '$ExamplesRepoName'" @@ -47,6 +48,8 @@ try { Write-Output "Testing Examples" mvn clean test "-DTestResourceKey=$($Keys.TestResourceKey)" "-DSuperResourceKey=$($Keys.TestResourceKey)" "-DLicenseKey=$($Keys.DeviceDetection)" + # Checked after the Selenium tests, so a failure here does not stop them. + $examplesStatus = $LASTEXITCODE Write-Output "Copying test results". # Copy the test results into the test-results folder @@ -73,36 +76,130 @@ finally { } -$status = $LASTEXITCODE +# Runs the shared Selenium contract tests (category Contract in +# selenium-api-tests) against one web example. The example is built, started +# in the background on its own port and left to serve until the tests finish. +# Any failure is added to $Failures rather than thrown, so that one example +# failing does not stop the other from being tested. +function Invoke-ContractTests { + param( + [Parameter(Mandatory)][string]$Label, + [Parameter(Mandatory)][string]$Module, + [Parameter(Mandatory)][int]$Port, + [Parameter(Mandatory)][AllowEmptyCollection()] + [System.Collections.Generic.List[string]]$Failures, + # Environment variables the example needs, removed again afterwards. + [hashtable]$ExampleEnv = @{} + ) + Write-Host "Running Selenium contract tests against the $Label example on port $Port" + $example = $null + $failure = $null + $stdout = Join-Path (Get-Location).Path "example-$Port.out.log" + $stderr = Join-Path (Get-Location).Path "example-$Port.err.log" + try { + # Build and start the example (already pinned to the dev version above). + # The example finds its web content relative to the examples repository + # root, so the process is started from there. + Push-Location "device-detection-java-examples" + try { + mvn -B --no-transfer-progress -pl $Module -am package -DskipTests | Out-Host + if ($LASTEXITCODE -ne 0) { + $failure = "the $Label example failed to build" + } else { + $jar = (Get-ChildItem "$Module/target/*-jar-with-dependencies.jar" | Select-Object -First 1).FullName + $env:PORT = $Port + foreach ($name in $ExampleEnv.Keys) { + Set-Item "env:$name" $ExampleEnv[$name] + } + $example = Start-Process java -ArgumentList '-jar', "`"$jar`"" ` + -WorkingDirectory (Get-Location).Path -NoNewWindow -PassThru ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr + } + } finally { Pop-Location } + + if (-not $failure) { + # Wait up to three minutes for the example to answer. + $up = $false + for ($i = 0; $i -lt 60 -and -not $up; $i++) { + try { + Invoke-WebRequest -UseBasicParsing -TimeoutSec 10 "http://localhost:$Port" | Out-Null + $up = $true + } catch { + if ($example.HasExited) { break } + Start-Sleep -Seconds 3 + } + } + if (-not $up) { + $failure = "the $Label example did not answer on port $Port" + } + } + + if (-not $failure) { + $env:EXAMPLE_URL = "http://localhost:$Port" + $env:EXAMPLE_LANG = 'java' + dotnet test selenium-api-tests -c Release --filter TestCategory=Contract | Out-Host + if ($LASTEXITCODE -ne 0) { + $failure = "the contract tests failed against the $Label example" + } + } + + if ($failure) { + $Failures.Add($failure) + if ($example) { + Write-Host ">>> $Label example output >>>" + Get-Content $stdout, $stderr -ErrorAction SilentlyContinue | Out-Host + Write-Host "<<< $Label example output <<<" + } + } + } finally { + if ($example -and -not $example.HasExited) { + Stop-Process -Id $example.Id -Force + } + Remove-Item env:PORT, env:EXAMPLE_URL -ErrorAction SilentlyContinue + foreach ($name in $ExampleEnv.Keys) { + Remove-Item "env:$name" -ErrorAction SilentlyContinue + } + } +} + +# Failures are collected so that every set of tests runs before the job fails. +$failures = [System.Collections.Generic.List[string]]::new() +if ($examplesStatus -ne 0) { + $failures.Add("the example tests failed with exit code $examplesStatus") +} Write-Host 'Running Selenium tests...' -try { - # Build and start the cloud example (already pinned to the dev version above). - Push-Location "device-detection-java-examples" - try { - mvn -B --no-transfer-progress -pl web/getting-started.cloud -am package -DskipTests - $jar = (Get-ChildItem web/getting-started.cloud/target/*-jar-with-dependencies.jar | Select-Object -First 1).FullName - $env:PORT = 8099 - $env:TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" - $env:TestResourceKey = $Keys.TestResourceKey - $example = java -jar $jar 2>&1 & - } finally { Pop-Location } - - # Get the shared contract tests. +if ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture -eq 'Arm64') { + # The Selenium Manager binary that the suite ships for Linux is built for + # x64 only, so on an Arm64 Linux runner every test fails with + # "Exec format error" before a browser is started. + Write-Host 'Skipping Selenium tests, Selenium Manager has no Linux Arm64 build' +} else { + # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of + # the suite, so a change there can be tried here before it is merged. + $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } if (-not (Test-Path selenium-api-tests)) { - git clone --depth 1 https://github.com/51Degrees/selenium-api-tests.git + Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" + git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git + if ($LASTEXITCODE -ne 0) { throw "Failed to clone selenium-api-tests" } } - # Wait for the example to come up. - curl -sS -o /dev/null --retry 5 --retry-connrefused "http://localhost:$env:PORT" - $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" $env:PAID_RESOURCE_KEY = $Keys.TestResourceKey - $env:EXAMPLE_URL = "http://localhost:$env:PORT" - $env:EXAMPLE_LANG = 'java' - dotnet test selenium-api-tests -c Release --filter TestCategory=Contract -} catch { - if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } - throw -} finally { - if ($example) { Remove-Job -Force $example } + + # The cloud example. + Invoke-ContractTests -Label 'cloud' -Module 'web/getting-started.cloud' -Port 8099 -Failures $failures -ExampleEnv @{ + TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" + TestResourceKey = $Keys.TestResourceKey + } + + # The on-premise example, against the TAC data file copied into the + # examples repository above. The Lite data file has neither DeviceType nor + # the JavaScript properties the contract tests need. + Invoke-ContractTests -Label 'on-premise' -Module 'web/getting-started.onprem' -Port 8098 -Failures $failures -ExampleEnv @{ + '51DEGREES_DD_PATH' = (Resolve-Path "device-detection-java-examples/device-detection-data/TAC-HashV41.hash").Path + } +} + +if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" }