diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ae705f41..99621cbd 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -73,67 +73,107 @@ if ($Version) { Write-Host "No version given, so the examples were tested by the unit tests against the reactor." } -# The shared Selenium contract tests, run against the Device Detection cloud -# web example. +# The shared Selenium contract tests, run against one web example. The example +# is built, started in the background on its own port and left to serve until +# the tests finish. Every web example, cloud and on-premise, is expected to give +# the same client-side results, so each is tested the same way. function Invoke-ContractTests { - $port = 8099 + param( + [Parameter(Mandatory)][string]$Label, + [Parameter(Mandatory)][string]$Module, + [Parameter(Mandatory)][int]$Port, + # Environment variables the example needs, removed again afterwards. + [hashtable]$ExampleEnv = @{} + ) + Write-Host "Running the Selenium contract tests against the $Label example on port $Port" $example = $null Push-Location "$RepoName/examples/device-detection" try { - mvn @MavenOptions -pl web/getting-started.cloud -am package -DskipTests @versionArgs - if ($LASTEXITCODE -ne 0) { throw "Building the cloud web example failed." } - $jar = (Get-ChildItem web/getting-started.cloud/target/*-jar-with-dependencies.jar | Select-Object -First 1).FullName - $env:PORT = $port - $env:TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" - $env:TestResourceKey = $TestResourceKey + mvn @MavenOptions -pl $Module -am package -DskipTests @versionArgs + if ($LASTEXITCODE -ne 0) { throw "Building the $Label web example failed." } + $jar = (Get-ChildItem "$Module/target/*-jar-with-dependencies.jar" | Select-Object -First 1).FullName + $env:PORT = $Port + foreach ($name in $ExampleEnv.Keys) { + Set-Item "env:$name" $ExampleEnv[$name] + } $example = java -jar $jar 2>&1 & } finally { Pop-Location } try { - # SELENIUM_TESTS_REF picks a branch of the suite, so a change there - # can be tried here before it is merged. - $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } - if (-not (Test-Path selenium-api-tests)) { - Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" - git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git - } - # Wait for the example to come up. - curl -sS -o $(if ($IsWindows) { 'NUL' } else { '/dev/null' }) --retry 20 --retry-delay 3 --retry-max-time 120 --retry-connrefused "http://localhost:$port" - if ($LASTEXITCODE -ne 0) { throw "The cloud web example did not start on port $port." } + curl -sS -o $(if ($IsWindows) { 'NUL' } else { '/dev/null' }) --retry 20 --retry-delay 3 --retry-max-time 120 --retry-connrefused "http://localhost:$Port" + if ($LASTEXITCODE -ne 0) { throw "The $Label web example did not start on port $Port." } - $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" - $env:PAID_RESOURCE_KEY = $TestResourceKey - $env:EXAMPLE_URL = "http://localhost:$port" + $env:EXAMPLE_URL = "http://localhost:$Port" $env:EXAMPLE_LANG = 'java' dotnet test selenium-api-tests -c Release --filter TestCategory=Contract # PowerShell does not stop for a native command that returns non-zero, # so without this the browser tests could fail every case and the job # would still be reported as a success. if ($LASTEXITCODE -ne 0) { - throw "The Selenium contract tests failed, returning $LASTEXITCODE. Their output is above." + throw "The Selenium contract tests failed against the $Label example, returning $LASTEXITCODE. Their output is above." } } catch { - if ($example) { Write-Host '>>> example app output >>>'; Receive-Job $example | Out-Host; Write-Host '<<< app output <<<' } + if ($example) { Write-Host ">>> $Label example output >>>"; Receive-Job $example | Out-Host; Write-Host "<<< $Label example output <<<" } throw } finally { if ($example) { Remove-Job -Force $example } + Remove-Item env:PORT, env:EXAMPLE_URL -ErrorAction SilentlyContinue + foreach ($name in $ExampleEnv.Keys) { + Remove-Item "env:$name" -ErrorAction SilentlyContinue + } } } Write-Host "::group::Selenium contract tests" if (-not $TestResourceKey) { - Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the cloud example." + Write-Host "::warning title=Selenium skipped::No resource key, so the Selenium contract tests were not run against the web examples." } elseif ($IsLinux -and [System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture -eq [System.Runtime.InteropServices.Architecture]::Arm64) { Write-Host "::warning title=Selenium skipped::Selenium contract skipped on linux-arm64, which has no selenium-manager build. It runs on x64 and macOS arm64." } else { - try { - Invoke-ContractTests - } catch { - Write-Host "::error title=Selenium contract failed::$_" - $failed += 'Selenium contract tests' + # Get the shared contract tests. SELENIUM_TESTS_REF picks a branch of the + # suite, so a change there can be tried here before it is merged. + $seleniumRef = if ($env:SELENIUM_TESTS_REF) { $env:SELENIUM_TESTS_REF } else { 'main' } + if (-not (Test-Path selenium-api-tests)) { + Write-Host "Cloning the Selenium contract tests from '$seleniumRef'" + git clone --depth 1 --branch $seleniumRef https://github.com/51Degrees/selenium-api-tests.git + } + $env:CLOUD_ROOT_URL = "https://cloud.51degrees.com/" + $env:PAID_RESOURCE_KEY = $TestResourceKey + + $contracts = @( + @{ + Label = 'cloud' + Module = 'web/getting-started.cloud' + Port = 8099 + ExampleEnv = @{ + TestCloudEndpoint = "https://cloud.51degrees.com/api/v4" + TestResourceKey = $TestResourceKey + } + } + # The on-premise example, against the TAC data file that + # fetch-assets.ps1 links into the data repository. The Lite data file + # has neither DeviceType nor the JavaScript properties the contract + # tests need. + @{ + Label = 'on-premise' + Module = 'web/getting-started.onprem' + Port = 8098 + ExampleEnv = @{ + '51DEGREES_DD_PATH' = (Resolve-Path "$RepoName/cxx/data/device-detection/TAC-HashV41.hash").Path + } + } + ) + # Each example is tested whether or not the one before it failed. + foreach ($contract in $contracts) { + try { + Invoke-ContractTests @contract + } catch { + Write-Host "::error title=Selenium contract failed::$_" + $failed += "Selenium contract tests, $($contract.Label) example" + } } } Write-Host "::endgroup::"