Release candidate

pipeline/.github/workflows/cloud-51did.yml → .github/workflows/cloud-51did.yml

@@ -1,20 +1,22 @@
 name: Cloud 51Did Tests
 
 # Runs the live cloud 51Did integration test
-# (fiftyone_pipeline_did/tests/test_did_client_live.py) once for every
-# _51DEGREES_RESOURCE_KEY* key passed in below. Each run sets the single
-# _51DEGREES_RESOURCE_KEY the test reads.
+# (pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py) once for
+# every _51DEGREES_RESOURCE_KEY* key passed to its step below. Each run sets
+# the single _51DEGREES_RESOURCE_KEY the test reads first. Those keys carry
+# the FODid product and the CloudV5FODiD product the marketing usages need.
+# The Basic key carries FODid without CloudV5FODiD, so it runs only the
+# tests of what the Basic tier is entitled to (test_did_tier_live.py).
 #
 # The secrets are named one by one rather than passed as toJSON(secrets).
 # GitHub marks a workflow that hands every secret to a step as possibly
 # malicious and holds each run for manual approval, so the nightly never ran.
 # A new resource key therefore needs a line here as well as the organisation
 # secret.
 #
-# The test skips itself when no key is set, so a run with no secrets is green
-# and proves nothing. That is deliberate, because a pull request from a fork
-# has no secrets, and it is the reason the loop below reports which key each
-# run used.
+# A run with no key fails, because a cloud test that passes without calling
+# the cloud has proved nothing. A pull request from a fork gets no secrets,
+# so run this workflow from a branch in this repository.
 
 on:
   workflow_dispatch:
@@ -29,8 +31,8 @@ on:
   pull_request:
     branches: [main]
     paths:
-      - 'fiftyone_pipeline_did/**'
-      - 'owid-python/**'
+      - 'pipeline/fiftyone_pipeline_did/**'
+      - 'pipeline/owid-python/**'
       - '.github/workflows/cloud-51did.yml'
 
 jobs:
@@ -39,7 +41,7 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: fiftyone_pipeline_did
+        working-directory: pipeline/fiftyone_pipeline_did
     steps:
       - uses: actions/checkout@v4
         with:
@@ -56,26 +58,27 @@ jobs:
         # runs.
         shell: pwsh
         working-directory: .
-        run: ./ci/copy-owid-source.ps1
+        run: ./pipeline/ci/copy-owid-source.ps1
       - name: Install tox
         run: python -m pip install --upgrade pip tox
       - name: Cloud 51Did test for each resource key secret
         shell: pwsh
         env:
-          _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }}
           _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }}
           _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }}
-          # The redemption tests skip without a license key, and the service
-          # only opens a sealed result for a license key of the account whose
-          # resource key sealed it. So each license key is passed only with
-          # the resource key of the same suffix.
+          # The redemption tests check a sealed result reads back only with a
+          # license key, and the service only opens a sealed result for a
+          # license key of the account whose resource key sealed it. So each
+          # license key is passed only with the resource key of the same
+          # suffix, and a resource key without one checks that the result
+          # redeems as unreadable.
           _51DEGREES_LICENSE_KEY_51DID: ${{ secrets._51DEGREES_LICENSE_KEY_51DID }}
         run: |
           $keys = Get-ChildItem env: |
             Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value }
           if (-not $keys) {
-            Write-Host 'No _51DEGREES_RESOURCE_KEY* secrets available; skipping.'
-            exit 0
+            Write-Host '::error::No _51DEGREES_RESOURCE_KEY* secret reached this job. The tests need a key that carries the FODid product, and the CloudV5FODiD product for the marketing usages.'
+            exit 1
           }
           $failed = $false
           foreach ($k in $keys) {
@@ -84,10 +87,14 @@ jobs:
             $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY'
             $env:_51DEGREES_LICENSE_KEY = [Environment]::GetEnvironmentVariable($licenseName)
             Write-Host "License key: $(if ($env:_51DEGREES_LICENSE_KEY) { $licenseName } else { 'none' })"
-            # -rs prints why each skipped test skipped, which tells a run that
-            # lacked a key apart from one the service turned away.
+            # -rs lists any test that skipped, and none should.
             tox -- tests/test_did_client_live.py -v -rs
             if ($LASTEXITCODE -ne 0) { $failed = $true }
             Write-Host '::endgroup::'
           }
           if ($failed) { exit 1 }
+      - name: Cloud 51Did tier test with the Basic key
+        if: ${{ !cancelled() }}
+        env:
+          _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }}
+        run: tox -- tests/test_did_tier_live.py -v -rs

ci/build-package.ps1

@@ -6,9 +6,12 @@ $ErrorActionPreference = "Stop"
 . (Join-Path $PSScriptRoot 'packages.ps1')
 
 # Builds the packages this publish releases, each at its own version from the
-# release plan. $Version is common-ci's label for the run and is not used.
+# release plan. When nothing is to be released, every package is built at its
+# released version, so that the tests have packages to install and the nightly
+# publish passes without publishing anything. $Version is common-ci's label for
+# the run and is not used.
 
-$plan = @(Get-ReleasePlan $RepoName)
+$plan = @(Get-BuildPlan $RepoName)
 
 # fiftyone_pipeline_did carries the OWID source inside itself, copied in by the
 # pipeline product's script; see pipeline/ci/copy-owid-source.ps1.

ci/packages.Tests.ps1

@@ -249,6 +249,36 @@ Describe 'Get-ReleasePlan' {
     }
 }
 
+Describe 'Get-BuildPlan' {
+    It 'is the release plan when a package is to be released' {
+        $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0')
+        Add-Commit $script:Repo @('pipeline/engines/a.py')
+        $plan = @(Get-BuildPlan $script:Repo $script:Graph)
+        $plan.Name | Should -Be @('engines', 'dd', 'ipi')
+        ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39'
+    }
+
+    It 'builds every published package at its released version when nothing changed' {
+        $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0')
+        Add-Commit $script:Repo @()
+        @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0
+        $plan = @(Get-BuildPlan $script:Repo $script:Graph)
+        $plan.Name | Should -Be @('core', 'did', 'engines', 'location', 'dd', 'ipi')
+        ($plan | Where-Object Name -eq 'core').Version | Should -Be '4.5.38'
+        ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.96'
+        ($plan | Where-Object Name -eq 'location').Version | Should -Be '4.5.4'
+        ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0'
+    }
+
+    It 'builds the released versions from a commit already released' {
+        $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0')
+        Add-Commit $script:Repo @('device/dd/a.py')
+        git -C $script:Repo tag 'dd/4.5.97'
+        ($plan = @(Get-BuildPlan $script:Repo $script:Graph)).Count | Should -Be 6
+        ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97'
+    }
+}
+
 Describe 'the real packages.json' {
     BeforeAll {
         $script:Real = Get-Packages (Split-Path $PSScriptRoot)
@@ -285,6 +315,11 @@ Describe 'the real packages.json' {
         $ipi.Count | Should -Be 5
         $ipi.FirstVersion | Select-Object -Unique | Should -Be '4.5.0'
     }
+
+    It 'has every tox.ini install the packages its package depends on' {
+        Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real |
+            Should -BeNullOrEmpty
+    }
 }
 
 }

ci/packages.ps1

@@ -30,6 +30,51 @@ function Test-PathUnder([string]$Path, [string]$Folder) {
     return $Path -eq $Folder -or $Path.StartsWith("$Folder/")
 }
 
+# Each tox.ini that does not install every package its package depends on,
+# directly or through another, from this checkout, as a line naming what is
+# missing. Taken from a package index instead, a dependency is tested at its
+# last release rather than with the code it is released with.
+function Get-ToxGaps([string]$RepoPath, $Packages = $null) {
+    if ($null -eq $Packages) {
+        $Packages = Get-Packages $RepoPath
+    }
+    $byName = @{}
+    foreach ($package in $Packages) {
+        $byName[$package.Name] = $package
+    }
+    $gaps = @()
+    foreach ($package in $Packages) {
+        $tox = Join-Path $RepoPath $package.Path 'tox.ini'
+        if ((Test-Path $tox) -eq $false) {
+            continue
+        }
+        $text = Get-Content -Raw $tox
+        $needed = [Collections.Generic.HashSet[string]]::new()
+        $queue = [Collections.Generic.Queue[string]]::new()
+        foreach ($name in $package.DependsOn) {
+            $queue.Enqueue($name)
+        }
+        while ($queue.Count -gt 0) {
+            $name = $queue.Dequeue()
+            if ($needed.Add($name) -eq $true) {
+                foreach ($next in $byName[$name].DependsOn) {
+                    $queue.Enqueue($next)
+                }
+            }
+        }
+        foreach ($name in $needed) {
+            $relative = [IO.Path]::GetRelativePath(
+                (Join-Path $RepoPath $package.Path),
+                (Join-Path $RepoPath $byName[$name].Path)).Replace('\', '/')
+            $pattern = '(?m)^\s*-e\s*' + [regex]::Escape($relative) + '/?\s*$'
+            if ($text -notmatch $pattern) {
+                $gaps += "$($package.Path)/tox.ini does not install $name from $relative"
+            }
+        }
+    }
+    return $gaps
+}
+
 # The names of the given packages and of every package that depends on one of
 # them, directly or through another.
 function Get-DependentClosure($Packages, [string[]]$Names) {
@@ -247,3 +292,29 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) {
         }
     }
 }
+
+# What a build of HEAD makes: the release plan, or, when nothing changed since
+# the last release, every published package at its released version. Nothing
+# is published then, because package-update-required.ps1 finds the release
+# plan empty, but the publish's tests still have the packages to install.
+function Get-BuildPlan([string]$RepoPath, $Packages = $null) {
+    if ($null -eq $Packages) {
+        $Packages = Get-Packages $RepoPath
+    }
+    $plan = @(Get-ReleasePlan $RepoPath $Packages)
+    if ($plan.Count -gt 0) {
+        return $plan
+    }
+    $published = @($Packages | Where-Object { $_.Publish -eq $true })
+    foreach ($name in (Sort-ByDependency $published @($published.Name))) {
+        $package = $published | Where-Object { $_.Name -eq $name }
+        $last = @(Get-ReleasedVersions $RepoPath $name)[-1]
+        [pscustomobject]@{
+            Name = $name
+            Path = $package.Path
+            Version = "$last"
+            Previous = $last
+            Reason = 'already released, built for the tests'
+        }
+    }
+}

ci/run-unit-tests.ps1

@@ -3,5 +3,15 @@ param (
 )
 $ErrorActionPreference = "Stop"
 . (Join-Path $PSScriptRoot 'dispatch.ps1')
+. (Join-Path $PSScriptRoot 'packages.ps1')
+
+# Each package's tests take the packages it depends on from this checkout, so
+# they test the code those packages are released with. A tox.ini that takes
+# one from PyPI instead stops the tests before they run.
+$gaps = @(Get-ToxGaps (Split-Path $PSScriptRoot))
+if ($gaps.Count -gt 0) {
+    throw "A tox.ini takes a package from PyPI that this checkout has:`n" +
+        ($gaps -join "`n")
+}
 
 Invoke-ProductStep 'run-unit-tests' $RepoName $PSBoundParameters

device-detection/ci/run-integration-tests.ps1

@@ -14,11 +14,14 @@ if ($IsWindows) {
 $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples"
 
 if (!$Keys.TestResourceKey) {
-    Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run."
-    return
-} elseif (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) {
-    Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so cloud tests will not run."
-    $packages = "fiftyone_devicedetection_examples"
+    throw "No TestResourceKey was passed, so the cloud tests cannot call " +
+        "the cloud service. The workflows set it from the " +
+        "_51DEGREES_RESOURCE_KEY_BESPOKE secret."
+}
+# The cloud package runs without the CSV file too, because only its property
+# coverage tests read the file, and they say so when it is missing.
+if (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) {
+    Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so the cloud property coverage tests will not run."
 }
 
 # nightly-publish-main workflow doesn't create the examples package, so

device-detection/ci/run-unit-tests.ps1

@@ -9,11 +9,20 @@ if ($IsWindows) {
     Write-Output $env:TEMP
 }
 
-# The cloud package's tests run against fixed cloud responses, so they need
-# no resource key, no licence key and no network connection. Running them
-# here means a broken cloud example shows as red on every build, rather than
-# only when the integration tests happen to have a key to run with.
-$packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise"
-./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages
+# The cloud package's tests against fixed cloud responses need no resource
+# key, no licence key and no network connection. Running them here means a
+# broken cloud example shows as red on every build, rather than only when the
+# integration tests happen to have a key to run with. Its tests marked live
+# call the cloud service, and no key reaches this step, so they are left to
+# run-integration-tests.ps1. Only a package whose tox.ini passes
+# PYTEST_ADDOPTS sees the selection.
+$addopts = $env:PYTEST_ADDOPTS
+$env:PYTEST_ADDOPTS = '-m "not live"'
+try {
+    $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise"
+    ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages
+} finally {
+    $env:PYTEST_ADDOPTS = $addopts
+}
 
 exit $LASTEXITCODE

device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py

@@ -83,18 +83,30 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs):
         @param licence_keys: licencekeys to use for the data file update service
         @type download: bool
         @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service
-        @type max_matched_useragents_length : int
-        @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false
-        @type update_matched_useragent: 
-        @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent
+        @type max_matched_useragent_length: int
+        @param max_matched_useragent_length: Limit in bytes on the length of
+        a single header such as User-Agent, of a pseudo header including the
+        separators between its values, and of the matched User-Agent. A value
+        longer than the limit is ignored and detection moves on to the next
+        header. The limit applies whether or not update_matched_useragent is
+        on. Left unset, the native default of 500 applies. A larger limit
+        raises the memory each set of results holds.
+        @type update_matched_useragent: bool
+        @param update_matched_useragent: True if the detection should record
+        the matched characters from the target User-Agent, which the
+        UserAgents property returns. On by default, as the native code sets
+        it. Set it to False when UserAgents is not needed, which saves
+        recording the characters.
         @type drift: int
         @param drift: Set maximum drift in hash position to allow when processing HTTP headers
         @type difference: int
         @param difference: set the maximum difference to allow when processing HTTP headers. The difference is the difference in hash value between the hash that was found, and the hash that is being searched for. By default this is 0.
         @type allow_unmatched : bool
         @param allow_unmatched:  True if there should be at least one matched node in order for the results to be considered valid. By default, this is false
         @type performance_profile: string
-        @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance
+        @param performance_profile: options are: LowMemory, MaxPerformance,
+        Balanced, BalancedTemp, HighPerformance. Balanced by default, as in
+        the native code.
         @type reuse_temp_file: bool
         @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage.
         @type concurrency: int

device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py

@@ -81,10 +81,6 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs):
         @param licence_keys: licencekeys to use for the data file update service
         @type download: bool
         @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service
-        @type max_matched_useragents_length : int
-        @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false
-        @type update_matched_useragent: 
-        @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent
         @type drift: int
         @param drift: Set maximum drift in hash position to allow when processing HTTP headers
         @type difference: int

device-detection/fiftyone_devicedetection_cloud/tests/conftest.py

@@ -0,0 +1,32 @@
+# *********************************************************************
+# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
+# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
+# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
+#
+# This Original Work is licensed under the European Union Public Licence
+# (EUPL) v.1.2 and is subject to its terms as set out below.
+#
+# If a copy of the EUPL was not distributed with this file, You can obtain
+# one at https://opensource.org/licenses/EUPL-1.2.
+#
+# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
+# amended by the European Commission) shall be deemed incompatible for
+# the purposes of the Work and the provisions of the compatibility
+# clause in Article 5 of the EUPL shall not apply.
+#
+# If using the Work as, or as part of, a network application, by
+# including the attribution notice(s) required under Article 5 of the EUPL
+# in the end user terms of the application under an appropriate heading,
+# such notice(s) shall fulfill the requirements of that article.
+# *********************************************************************
+
+# The marker that keeps the live cloud tests out of the unit test run, which
+# device-detection/ci/run-unit-tests.ps1 deselects. The tests against fixed
+# cloud responses carry no marker, so they run in both steps.
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "live: calls the live 51Degrees cloud service and needs a resource "
+        "key")