diff --git a/pipeline/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml similarity index 65% rename from pipeline/.github/workflows/cloud-51did.yml rename to .github/workflows/cloud-51did.yml index 95b128b4..5f258dc1 100644 --- a/pipeline/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -1,9 +1,12 @@ name: Cloud 51Did Tests # Runs the live cloud 51Did integration test -# (fiftyone_pipeline_did/tests/test_did_client_live.py) once for every -# _51DEGREES_RESOURCE_KEY* key passed in below. Each run sets the single -# _51DEGREES_RESOURCE_KEY the test reads. +# (pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py) once for +# every _51DEGREES_RESOURCE_KEY* key passed to its step below. Each run sets +# the single _51DEGREES_RESOURCE_KEY the test reads first. Those keys carry +# the FODid product and the CloudV5FODiD product the marketing usages need. +# The Basic key carries FODid without CloudV5FODiD, so it runs only the +# tests of what the Basic tier is entitled to (test_did_tier_live.py). # # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly @@ -11,10 +14,9 @@ name: Cloud 51Did Tests # A new resource key therefore needs a line here as well as the organisation # secret. # -# The test skips itself when no key is set, so a run with no secrets is green -# and proves nothing. That is deliberate, because a pull request from a fork -# has no secrets, and it is the reason the loop below reports which key each -# run used. +# A run with no key fails, because a cloud test that passes without calling +# the cloud has proved nothing. A pull request from a fork gets no secrets, +# so run this workflow from a branch in this repository. on: workflow_dispatch: @@ -29,8 +31,8 @@ on: pull_request: branches: [main] paths: - - 'fiftyone_pipeline_did/**' - - 'owid-python/**' + - 'pipeline/fiftyone_pipeline_did/**' + - 'pipeline/owid-python/**' - '.github/workflows/cloud-51did.yml' jobs: @@ -39,7 +41,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: fiftyone_pipeline_did + working-directory: pipeline/fiftyone_pipeline_did steps: - uses: actions/checkout@v4 with: @@ -56,26 +58,27 @@ jobs: # runs. shell: pwsh working-directory: . - run: ./ci/copy-owid-source.ps1 + run: ./pipeline/ci/copy-owid-source.ps1 - name: Install tox run: python -m pip install --upgrade pip tox - name: Cloud 51Did test for each resource key secret shell: pwsh env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} - # The redemption tests skip without a license key, and the service - # only opens a sealed result for a license key of the account whose - # resource key sealed it. So each license key is passed only with - # the resource key of the same suffix. + # The redemption tests check a sealed result reads back only with a + # license key, and the service only opens a sealed result for a + # license key of the account whose resource key sealed it. So each + # license key is passed only with the resource key of the same + # suffix, and a resource key without one checks that the result + # redeems as unreadable. _51DEGREES_LICENSE_KEY_51DID: ${{ secrets._51DEGREES_LICENSE_KEY_51DID }} run: | $keys = Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } if (-not $keys) { - Write-Host 'No _51DEGREES_RESOURCE_KEY* secrets available; skipping.' - exit 0 + Write-Host '::error::No _51DEGREES_RESOURCE_KEY* secret reached this job. The tests need a key that carries the FODid product, and the CloudV5FODiD product for the marketing usages.' + exit 1 } $failed = $false foreach ($k in $keys) { @@ -84,10 +87,14 @@ jobs: $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY' $env:_51DEGREES_LICENSE_KEY = [Environment]::GetEnvironmentVariable($licenseName) Write-Host "License key: $(if ($env:_51DEGREES_LICENSE_KEY) { $licenseName } else { 'none' })" - # -rs prints why each skipped test skipped, which tells a run that - # lacked a key apart from one the service turned away. + # -rs lists any test that skipped, and none should. tox -- tests/test_did_client_live.py -v -rs if ($LASTEXITCODE -ne 0) { $failed = $true } Write-Host '::endgroup::' } if ($failed) { exit 1 } + - name: Cloud 51Did tier test with the Basic key + if: ${{ !cancelled() }} + env: + _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} + run: tox -- tests/test_did_tier_live.py -v -rs diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index f806b1f7..057d3167 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -6,9 +6,12 @@ $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'packages.ps1') # Builds the packages this publish releases, each at its own version from the -# release plan. $Version is common-ci's label for the run and is not used. +# release plan. When nothing is to be released, every package is built at its +# released version, so that the tests have packages to install and the nightly +# publish passes without publishing anything. $Version is common-ci's label for +# the run and is not used. -$plan = @(Get-ReleasePlan $RepoName) +$plan = @(Get-BuildPlan $RepoName) # fiftyone_pipeline_did carries the OWID source inside itself, copied in by the # pipeline product's script; see pipeline/ci/copy-owid-source.ps1. diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 77ecd47e..766d1380 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -249,6 +249,36 @@ Describe 'Get-ReleasePlan' { } } +Describe 'Get-BuildPlan' { + It 'is the release plan when a package is to be released' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('pipeline/engines/a.py') + $plan = @(Get-BuildPlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('engines', 'dd', 'ipi') + ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39' + } + + It 'builds every published package at its released version when nothing changed' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @() + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + $plan = @(Get-BuildPlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('core', 'did', 'engines', 'location', 'dd', 'ipi') + ($plan | Where-Object Name -eq 'core').Version | Should -Be '4.5.38' + ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.96' + ($plan | Where-Object Name -eq 'location').Version | Should -Be '4.5.4' + ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0' + } + + It 'builds the released versions from a commit already released' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/a.py') + git -C $script:Repo tag 'dd/4.5.97' + ($plan = @(Get-BuildPlan $script:Repo $script:Graph)).Count | Should -Be 6 + ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' + } +} + Describe 'the real packages.json' { BeforeAll { $script:Real = Get-Packages (Split-Path $PSScriptRoot) @@ -285,6 +315,11 @@ Describe 'the real packages.json' { $ipi.Count | Should -Be 5 $ipi.FirstVersion | Select-Object -Unique | Should -Be '4.5.0' } + + It 'has every tox.ini install the packages its package depends on' { + Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | + Should -BeNullOrEmpty + } } } diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 5043d0aa..429c43ff 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -30,6 +30,51 @@ function Test-PathUnder([string]$Path, [string]$Folder) { return $Path -eq $Folder -or $Path.StartsWith("$Folder/") } +# Each tox.ini that does not install every package its package depends on, +# directly or through another, from this checkout, as a line naming what is +# missing. Taken from a package index instead, a dependency is tested at its +# last release rather than with the code it is released with. +function Get-ToxGaps([string]$RepoPath, $Packages = $null) { + if ($null -eq $Packages) { + $Packages = Get-Packages $RepoPath + } + $byName = @{} + foreach ($package in $Packages) { + $byName[$package.Name] = $package + } + $gaps = @() + foreach ($package in $Packages) { + $tox = Join-Path $RepoPath $package.Path 'tox.ini' + if ((Test-Path $tox) -eq $false) { + continue + } + $text = Get-Content -Raw $tox + $needed = [Collections.Generic.HashSet[string]]::new() + $queue = [Collections.Generic.Queue[string]]::new() + foreach ($name in $package.DependsOn) { + $queue.Enqueue($name) + } + while ($queue.Count -gt 0) { + $name = $queue.Dequeue() + if ($needed.Add($name) -eq $true) { + foreach ($next in $byName[$name].DependsOn) { + $queue.Enqueue($next) + } + } + } + foreach ($name in $needed) { + $relative = [IO.Path]::GetRelativePath( + (Join-Path $RepoPath $package.Path), + (Join-Path $RepoPath $byName[$name].Path)).Replace('\', '/') + $pattern = '(?m)^\s*-e\s*' + [regex]::Escape($relative) + '/?\s*$' + if ($text -notmatch $pattern) { + $gaps += "$($package.Path)/tox.ini does not install $name from $relative" + } + } + } + return $gaps +} + # The names of the given packages and of every package that depends on one of # them, directly or through another. function Get-DependentClosure($Packages, [string[]]$Names) { @@ -247,3 +292,29 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { } } } + +# What a build of HEAD makes: the release plan, or, when nothing changed since +# the last release, every published package at its released version. Nothing +# is published then, because package-update-required.ps1 finds the release +# plan empty, but the publish's tests still have the packages to install. +function Get-BuildPlan([string]$RepoPath, $Packages = $null) { + if ($null -eq $Packages) { + $Packages = Get-Packages $RepoPath + } + $plan = @(Get-ReleasePlan $RepoPath $Packages) + if ($plan.Count -gt 0) { + return $plan + } + $published = @($Packages | Where-Object { $_.Publish -eq $true }) + foreach ($name in (Sort-ByDependency $published @($published.Name))) { + $package = $published | Where-Object { $_.Name -eq $name } + $last = @(Get-ReleasedVersions $RepoPath $name)[-1] + [pscustomobject]@{ + Name = $name + Path = $package.Path + Version = "$last" + Previous = $last + Reason = 'already released, built for the tests' + } + } +} diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index 77ad8cad..e74a60ff 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -3,5 +3,15 @@ param ( ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') +. (Join-Path $PSScriptRoot 'packages.ps1') + +# Each package's tests take the packages it depends on from this checkout, so +# they test the code those packages are released with. A tox.ini that takes +# one from PyPI instead stops the tests before they run. +$gaps = @(Get-ToxGaps (Split-Path $PSScriptRoot)) +if ($gaps.Count -gt 0) { + throw "A tox.ini takes a package from PyPI that this checkout has:`n" + + ($gaps -join "`n") +} Invoke-ProductStep 'run-unit-tests' $RepoName $PSBoundParameters diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index f1483528..c2d33cc4 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -14,11 +14,14 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run." - return -} elseif (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so cloud tests will not run." - $packages = "fiftyone_devicedetection_examples" + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." +} +# The cloud package runs without the CSV file too, because only its property +# coverage tests read the file, and they say so when it is missing. +if (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so the cloud property coverage tests will not run." } # nightly-publish-main workflow doesn't create the examples package, so diff --git a/device-detection/ci/run-unit-tests.ps1 b/device-detection/ci/run-unit-tests.ps1 index 7bb9dea9..4acebce4 100644 --- a/device-detection/ci/run-unit-tests.ps1 +++ b/device-detection/ci/run-unit-tests.ps1 @@ -9,11 +9,20 @@ if ($IsWindows) { Write-Output $env:TEMP } -# The cloud package's tests run against fixed cloud responses, so they need -# no resource key, no licence key and no network connection. Running them -# here means a broken cloud example shows as red on every build, rather than -# only when the integration tests happen to have a key to run with. -$packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise" -./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +# The cloud package's tests against fixed cloud responses need no resource +# key, no licence key and no network connection. Running them here means a +# broken cloud example shows as red on every build, rather than only when the +# integration tests happen to have a key to run with. Its tests marked live +# call the cloud service, and no key reaches this step, so they are left to +# run-integration-tests.ps1. Only a package whose tox.ini passes +# PYTEST_ADDOPTS sees the selection. +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "not live"' +try { + $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise" + ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py index 13a1f624..1a735f87 100644 --- a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py @@ -83,10 +83,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -94,7 +104,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py index 4f59cada..8380803d 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py @@ -81,10 +81,6 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py b/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py new file mode 100644 index 00000000..8890cf53 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker that keeps the live cloud tests out of the unit test run, which +# device-detection/ci/run-unit-tests.ps1 deselects. The tests against fixed +# cloud responses carry no marker, so they run in both steps. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index c5369df1..177a42ed 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -24,33 +24,20 @@ import os import time +import pytest + from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud -# Resource key environment variables follow the 51Degrees convention, which -# is that every one of them starts with "_51DEGREES_RESOURCE_KEY". The name -# used before the convention was adopted is still read, so an existing setup -# keeps working. -RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" -LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - -MISSING_KEY_MESSAGE = ( - "No resource key found, so the tests that call the cloud service " - f"cannot run. Set the environment variable '{RESOURCE_KEY_ENV_VAR}' " - f"(the older name '{LEGACY_RESOURCE_KEY_ENV_VAR}' is still read). " - "Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-tests-test_cloud.py&utm_term=resource-key-required") - +from .live_cloud import find_resource_key, no_resource_key -def get_resource_key(): - """! - The resource key from the environment, or None when neither the aligned - nor the older variable is set. - """ +pytestmark = pytest.mark.live - return (os.environ.get(RESOURCE_KEY_ENV_VAR) - or os.environ.get(LEGACY_RESOURCE_KEY_ENV_VAR)) +NO_KEY = no_resource_key( + "the device product", + "device detection properties such as IsMobile, and their metadata", + several=True) mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" @@ -59,10 +46,10 @@ def get_resource_key(): class DeviceDetectionTests(unittest.TestCase): def setUp(self): - self.resource_key = get_resource_key() + self.resource_key = find_resource_key() if not self.resource_key: - self.skipTest(MISSING_KEY_MESSAGE) + self.fail(NO_KEY) def test_pipeline_builder_cloud_engine_init(self): diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py index 43bbeb34..8a525e71 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py @@ -23,6 +23,8 @@ import os import unittest +import pytest + from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud @@ -32,6 +34,10 @@ is_same_type, ) +from .live_cloud import find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + header_file_path = "./tests/51Degrees.csv" mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile" @@ -43,71 +49,54 @@ # TODO remove setheader properties from this list once UACH datafile is released. exclude_properties = ["setheaderbrowseraccept-ch", "setheaderplatformaccept-ch", "setheaderhardwareaccept-ch"] -# Resource key environment variables follow the 51Degrees convention, which -# is that every one of them starts with "_51DEGREES_RESOURCE_KEY". The name -# used before the convention was adopted is still read, so an existing setup -# keeps working. -RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" -LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - -resource_key = (os.environ.get(RESOURCE_KEY_ENV_VAR) - or os.environ.get(LEGACY_RESOURCE_KEY_ENV_VAR)) - -if not resource_key: - # Skipping rather than raising, so a run without a key shows as skipped - # and names the variable, instead of turning into a collection error. - import pytest - - pytest.skip( - "No resource key found, so the tests that call the cloud service " - f"cannot run. Set the environment variable '{RESOURCE_KEY_ENV_VAR}' " - f"(the older name '{LEGACY_RESOURCE_KEY_ENV_VAR}' is still read). " - "Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-tests-test_properties.py&utm_term=resource-key-required", - allow_module_level=True) - -if not os.path.isfile(header_file_path): - # The property list comes from an asset the build fetches with a licence - # key. Without it these tests cannot run, so skip rather than turn a - # missing asset into a failure. - import pytest - - pytest.skip( - f"The property list file '{header_file_path}' is not present, so " - "the property coverage tests cannot run. It is fetched by " - "ci/fetch-assets.ps1 when a device detection licence key is " - "available.", - allow_module_level=True) - -# Get Properties list -properties_list = get_properties_from_header_file(header_file_path) - -# Create a simple pipeline to access the engine with and process it with flow data -cloudRequestEngine = CloudRequestEngine({"resource_key": resource_key}) -deviceDetectionCloudEngine = DeviceDetectionCloud() -pipeline = PipelineBuilder() \ - .add(cloudRequestEngine) \ - .add(deviceDetectionCloudEngine) \ - .build() - +NO_KEY = no_resource_key( + "the device product", + "every device detection property listed in " + header_file_path, + several=True) + class PropertyTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.resource_key = find_resource_key() + cls.pipeline = None + if cls.resource_key and os.path.isfile(header_file_path): + cls.properties_list = get_properties_from_header_file( + header_file_path) + cls.engine = DeviceDetectionCloud() + cls.pipeline = PipelineBuilder() \ + .add(CloudRequestEngine({"resource_key": cls.resource_key})) \ + .add(cls.engine) \ + .build() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + if self.pipeline is None: + # The property list comes from an asset the build fetches with + # a licence key, so a run without it says which file is missing. + self.skipTest( + f"The property list file '{header_file_path}' is not " + "present, so the property coverage tests cannot run. It is " + "fetched by ci/fetch-assets.ps1 when a device detection " + "licence key is available.") + def test_available_properties(self): """! Tests whether the all the properties present in the engine when initialised with a resource key are accessible. """ - flowData = pipeline.create_flowdata() + flowData = self.pipeline.create_flowdata() flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() - elementData = flowData.get(deviceDetectionCloudEngine.datakey) + elementData = flowData.get(self.engine.datakey) # Get dictionary of elementdata properties dd_property_dict = elementData.as_dictionary() #RUn test checks on all the properties available in header file - for property in properties_list: + for property in self.properties_list: property = property.lower() if(property not in exclude_properties): if(property in dd_property_dict): @@ -128,16 +117,16 @@ def test_value_types(self): Tests value types of the properties present present in the engine """ - flowData = pipeline.create_flowdata() + flowData = self.pipeline.create_flowdata() flowData.evidence.add("header.user-agent", chrome_ua) flowData.process() - elementData = flowData.get(deviceDetectionCloudEngine.datakey) + elementData = flowData.get(self.engine.datakey) # Get dictionary of elementdata properties dd_property_dict = elementData.as_dictionary() # Get properties from the engine - properties_list = deviceDetectionCloudEngine.get_properties() + properties_list = self.engine.get_properties() # Run test check valuetypes of properties for propertykey, propertymeta in properties_list.items(): diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index 14fad039..c58bf36f 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -12,13 +12,23 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. -e../fiftyone_devicedetection_shared/ + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_devicedetection_cloud {tty:--color=yes} {posargs} +# PYTEST_ADDOPTS carries the marker selection of ../ci/run-unit-tests.ps1, +# which leaves the live tests out of the unit step. pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key + PYTEST_ADDOPTS [testenv:pre-publish] package = skip diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index c668d1d5..846ebc97 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -16,13 +16,35 @@ getting started example: `python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console` +The on-premise examples use the free Lite data file by default. The data +repository, at `cxx/data`, links to it, and an example downloads it the +first time it is missing, so the examples run from a fresh clone with no +settings. Where an example shows a property that only the Enterprise data +file has, such as HardwareVendor, HardwareName or DeviceType, it shows the +property as needing the Enterprise data file and gives a link to +[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=enterprise-data-file) +about a copy. Supply the path to an Enterprise data file as the first +command line argument to see every property. + +The data file update example needs a licence key, and the hardware profile +examples need the TAC data file or its CSV export, so without them each one +says what it needs and where to ask for it, and stops. + ## Tests To run the tests use: `python -m unittest discover -s tests -p test*.py -b` -Cloud tests will only run with a valid 51Degrees resource key (see above) set as a resource_key operating system environment variable. +The tests in `tests/test_examples_without_settings.py` run the on-premise +examples as they run from a fresh clone, with no licence key, and check that +each one finishes on the free Lite data file and gives the link to ask for +the Enterprise data file. + +The cloud tests need a valid 51Degrees resource key (see above), and fail +without one. They read it from the environment variable +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`, then +`resource_key`. For example, use following command to set resource_key on Linux: `export resource_key=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index 05260f95..2a50c478 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -24,6 +24,7 @@ from threading import Event, Thread import os import shlex +import shutil import subprocess from pathlib import Path @@ -59,9 +60,16 @@ class ExampleUtils: DATA_FOLDER = os.path.join("cxx", "data", "device-detection") # Sent with each download, so the downloads these examples make can be - # counted. + # counted. It is left out inside GitHub Actions, where the data + # repository's script names the build instead, so that test runs are + # not counted as people using the examples. DOWNLOAD_SOURCE = "examples-51degrees-python" + # Shown in place of a value for a property that the data file in use + # does not have. Every property the examples ask for is in the + # Enterprise data file. + NEEDS_ENTERPRISE = "needs the Enterprise data file" + @staticmethod def get_resource_key_from_config(config): key = "" @@ -246,14 +254,16 @@ def __find_file(file_name, stop_event, result, search_dir=None): if search_dir is None: search_dir = Path(os.getcwd()).absolute() - # Search for the file in the current directory - files = list(search_dir.rglob(_file.name)) - - # If the file is found in the current directory - # then set the result and return - if len(files): - result[0] = str(files[0].absolute()) - return + # Search the current directory and every folder below it. The stop + # event is checked in each folder, so the search ends soon after it + # times out, and a folder that cannot be read, such as one under + # /proc, is skipped. + for folder, _, names in os.walk(search_dir): + if stop_event.is_set(): + return + if _file.name in names: + result[0] = str(Path(folder, _file.name).absolute()) + return # If the file is not found in the current directory # then move to the parent directory and repeat the process @@ -271,23 +281,34 @@ def __find_file(file_name, stop_event, result, search_dir=None): def __download(path): name = os.path.basename(path) data = os.path.dirname(os.path.dirname(path)) + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + environment = None if os.name == "nt": + # PowerShell 7 when it is on the path, otherwise Windows + # PowerShell, which, started from PowerShell 7, inherits its + # module path and then cannot load its own commands. + power_shell = "pwsh" if shutil.which("pwsh") else "powershell" command = [ - "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", - "-File", os.path.join(data, "get-data-files.ps1"), name, - "-Source", ExampleUtils.DOWNLOAD_SOURCE] + power_shell, "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name] + if not in_actions: + command += ["-Source", ExampleUtils.DOWNLOAD_SOURCE] + if power_shell == "powershell": + environment = dict(os.environ) + environment.pop("PSModulePath", None) text = subprocess.list2cmdline(command) else: - command = [ - "sh", os.path.join(data, "get-data-files.sh"), - "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + command = ["sh", os.path.join(data, "get-data-files.sh")] + if not in_actions: + command += ["-s", ExampleUtils.DOWNLOAD_SOURCE] + command.append(name) text = shlex.join(command) # Flushed first, because the script writes to the same output. print(f"{name} is missing, so it is being downloaded by running\n" f"{text}", flush=True) try: - subprocess.run(command, check=False) + subprocess.run(command, check=False, env=environment) except OSError: pass if os.path.isfile(path): @@ -310,8 +331,70 @@ def data_file_is_old(engine): def get_data_file_tier(engine): return engine.engine.getProduct() + @staticmethod + def is_lite_data_file(engine): + """! + True when the engine uses the free Lite data file. + """ + return ExampleUtils.get_data_file_tier(engine) == "Lite" + + @staticmethod + def has_property(engine, name): + """! + True when the engine's data file has the property. The Lite data + file leaves out the properties only the Enterprise data file has, + such as HardwareVendor, HardwareName and DeviceType. The data + file's own list is read, because the engine's properties are only + those it was built to give. + """ + properties = engine.engine.getMetaData().getProperties() + name = name.lower() + for index in range(properties.getSize()): + if properties.getByIndex(index).getName().lower() == name: + return True + return False + + @staticmethod + def get_enterprise_value(engine, device, property): + """! + Format a property for display, or say that it needs the Enterprise + data file when the engine's data file does not have it. The value + is only read when the data file has the property. + """ + if not ExampleUtils.has_property(engine, property): + return ExampleUtils.NEEDS_ENTERPRISE + return ExampleUtils.get_human_readable(device, property) + + @staticmethod + def get_enterprise_note(contact_us): + """! + Where to ask for a copy of the Enterprise data file, written after an + example has shown properties that the data file in use does not + have. The link is tagged with the example's file, so that an enquiry + can be traced to the example. + """ + return ("Some of these properties need the Enterprise data file. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def get_enterprise_offer(contact_us): + """! + What the Enterprise data file adds and where to ask for a copy, + written after an example that shows no properties has run on the + free Lite data file. + """ + return ("This example used the free Lite data file. The Enterprise " + "data file adds many more properties, such as " + "HardwareVendor, HardwareName and DeviceType. Contact us for " + f"a copy at {contact_us}") + @staticmethod def check_data_file(pipeline, logger): + """! + Log the data file in use, and a warning if it is old. An example run + on the free Lite data file says what the Enterprise data file adds + in its own output. + """ # Get the 'engine' element within the pipeline that # performs device detection. We can use this to get # details about the data file as well as meta-data @@ -337,14 +420,6 @@ def check_data_file(pipeline, logger): "cxx/data/get-data-files.sh -u 51Degrees-LiteV4.1.hash, " + "or cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash " + "-Update on Windows. " + - "Find out about the Enterprise data file, which " + - "includes automatic updates, on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") - - if ExampleUtils.get_data_file_tier(engine) == "Lite": - logger.log("warning", - "This example is using the 'Lite' " + - "data file. This is used for illustration, and " + - "has limited accuracy and capabilities. Find " + - "out about the Enterprise data file on our " + - "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=lite-data-file") + "Contact us about the Enterprise data file, which " + + "includes automatic updates, at " + + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py index da30fdbe..c4c78c15 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py @@ -40,7 +40,6 @@ import sys from fiftyone_devicedetection_examples.example_utils import ExampleUtils -from fiftyone_pipeline_core.logger import Logger # CSV file name to search for CSV_FILE_NAME = "51Degrees-Tac-HardwarePlatform.csv" @@ -52,6 +51,10 @@ # Columns whose values are pipe-delimited lists MULTI_VALUE_COLUMNS = ["TAC", "HardwareName", "NativeName", "NativeModel"] +# Where to ask for a copy of the hardware profile export, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-csv_profile_metadata.py&utm_term=enterprise-data-file" + def flush_print(*args, **kwargs): print(*args, **kwargs, flush=True) @@ -110,15 +113,17 @@ def main(argv): output_file = None if output_arg == "-" else output_arg limit = int(argv[2]) if len(argv) > 2 else None - logger = Logger(min_level="info") output_func = flush_print if output_file else stderr_print if csv_file is not None: run(csv_file, output_file, output_func, limit) else: - logger.log("error", - "Failed to find the CSV data file. " - "Please provide the path to 51Degrees-Tac-HardwarePlatform.csv as the first argument.") + # The export is not one of the free data files, so the example says + # what it needs and where to ask for it, and stops. + print(f"Listing the hardware profiles needs the {CSV_FILE_NAME} " + "export, which is not one of the free data files. Give its " + "path as the first command line argument, or contact us for a " + f"copy at {CONTACT_US}") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 4fd126fc..c6d0e6a5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -32,9 +32,9 @@ # - Daily auto-update # # ## License Key -# In order to test this example you will need a 51Degrees Enterprise license which can be -# purchased from our [pricing page](https://51degrees.com/pricing/annual?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top). Look for our "Bigger" or -# "Biggest" options. +# Updating the data file needs a 51Degrees licence key for the Enterprise data file. +# Without one, the example shows the free Lite data file and says where to ask for a key. +# Contact us for a key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top # # # Data Files # You can find out more about data files, licenses etc. at our [FAQ page](https://51degrees.com/resources/faqs?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top) @@ -166,10 +166,14 @@ from fiftyone_devicedetection_shared.key_utils import KeyUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +# Where to ask for a licence key and a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" DEFAULT_DATA_FILENAME = os.path.expanduser("~") + os.path.sep + ENTERPRISE_DATAFILE_NAME @@ -187,6 +191,32 @@ def clear(self): super().clear() class DataFileUpdateConsole(): + def explain_license_key(self, data_file, output): + # Without a licence key the example cannot download a data file, so + # it shows the data file it would start from, the free Lite file + # unless another is given, and says what an update needs. + if not data_file: + data_file = ExampleUtils.find_file(LITE_DATAFILE_NAME) + if data_file and os.path.isfile(data_file): + pipeline = DeviceDetectionPipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + with pipeline: + engine = pipeline.get_element("device") + output(f"The data file in use is the " + f"'{ExampleUtils.get_data_file_tier(engine)}' file published on " + f"{ExampleUtils.get_data_file_date(engine):%Y-%m-%d}, at " + f"'{data_file}'.") + output("") + output("Updating the data file downloads the Enterprise data file, " + "which needs a licence key. Give the key as the first command line " + f"argument, or in the environment variable " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key and a " + f"copy at {CONTACT_US}") + def run(self, data_file, license_key, interactive, logger, output): logger.log("info", "Starting example") @@ -195,13 +225,8 @@ def run(self, data_file, license_key, interactive, logger, output): license_key = KeyUtils.get_named_key(UPDATE_EXAMPLE_LICENSE_KEY_NAME) if not license_key: - logger.log("error", - "In order to test this example you will need a 51Degrees Enterprise " - "license which can be obtained on a trial basis or purchased from our\n" - "pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=license-key-required. You must supply the license " - "key as an argument to this program, or as an environment or system variable " - f"named '{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'") - raise Exception("No license key available") + self.explain_license_key(data_file, output) + return # work out where the downloaded file will be put, directory must exist if (data_file != None): @@ -283,7 +308,7 @@ def run(self, data_file, license_key, interactive, logger, output): # to notify when update complete data_file_update_service = update_service, # For automatic updates to work you will need to provide a license key. - # A license key can be obtained with a subscription from https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=run + # Contact us for a licence key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=run licence_keys = license_key, # Enable update on startup, the auto update system # will be used to check for an update before the diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 3675fee2..4742eed1 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -45,6 +45,17 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=enterprise-data-file" + +# The properties only the Enterprise data file has, with the names they are +# shown with. The free Lite data file has the ones the example shows first. +ENTERPRISE_PROPERTIES = [ + ("Hardware Vendor", "hardwarevendor"), + ("Hardware Name", "hardwarename"), + ("Device Type", "devicetype")] + class GettingStartedConsole(): def run(self, data_file, logger, output): @@ -54,9 +65,11 @@ def run(self, data_file, logger, output): # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, - # We use the low memory profile as its performance is - # sufficient for this example. See the documentation for - # more detail on this and other configuration options: + # We use the MaxPerformance profile, which loads the whole data + # file into memory. The free Lite data file is about 35 MB, so + # that costs little memory and gives the fastest detections. + # See the documentation for more detail on this and other + # configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run @@ -76,8 +89,12 @@ def run(self, data_file, logger, output): ExampleUtils.check_data_file(pipeline, logger) # carry out some sample detections + needs_enterprise = False for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + needs_enterprise |= self.analyseEvidence( + values, pipeline, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) def analyseEvidence(self, evidence, pipeline, output): @@ -126,7 +143,18 @@ def analyseEvidence(self, evidence, pipeline, output): self.outputValue("Platform Version", device.platformversion, message) self.outputValue("Browser Name", device.browsername, message) self.outputValue("Browser Version", device.browserversion, message) + + # The engine says which properties its data file has, and the + # value of a property it does not have is not read. + engine = pipeline.get_element("device") + needs_enterprise = False + for name, property in ENTERPRISE_PROPERTIES: + value = ExampleUtils.get_enterprise_value( + engine, device, property) + needs_enterprise |= value == ExampleUtils.NEEDS_ENTERPRISE + message.append(f"\t{name}: {value}\n") output("".join(message)) + return needs_enterprise def outputValue(self, name, value, message): # Individual result values have a wrapper called @@ -147,9 +175,10 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main + # limited accuracy and capabilities. The example shows the properties + # only the Enterprise data file has as needing it. Contact us for a copy + # of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html index d26ab8da..a1352ee8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html @@ -32,8 +32,8 @@ days old. A more recent data file may be needed to correctly detect the latest devices, browsers, etc. Get the latest lite data file with the scripts in the data - repository on GitHub. Find out about the Enterprise data file, which includes automatic - daily updates, on our pricing page. + repository on GitHub. Contact us + about the Enterprise data file, which includes automatic daily updates. {% endif %}

Web integration example

@@ -130,17 +130,17 @@

Device data

- Hardware Vendor:{{ utils.get_human_readable(data.device, "hardwarevendor") }} - Hardware Name:{{ utils.get_human_readable(data.device, "hardwarename") }} - Device Type:{{ utils.get_human_readable(data.device, "devicetype") }} - Platform Vendor:{{ utils.get_human_readable(data.device, "platformvendor") }} - Platform Name:{{ utils.get_human_readable(data.device, "platformname") }} - Platform Version:{{ utils.get_human_readable(data.device, "platformversion") }} - Browser Vendor:{{ utils.get_human_readable(data.device, "browservendor") }} - Browser Name:{{ utils.get_human_readable(data.device, "browsername") }} - Browser Version:{{ utils.get_human_readable(data.device, "browserversion") }} - Screen width (pixels):{{ utils.get_human_readable(data.device, "screenpixelswidth") }} - Screen height (pixels):{{ utils.get_human_readable(data.device, "screenpixelsheight") }} + Hardware Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarevendor") }} + Hardware Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarename") }} + Device Type:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "devicetype") }} + Platform Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformvendor") }} + Platform Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformname") }} + Platform Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformversion") }} + Browser Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browservendor") }} + Browser Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browsername") }} + Browser Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browserversion") }} + Screen width (pixels):{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "screenpixelswidth") }} + Screen height (pixels):{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "screenpixelsheight") }} {# The device id identifies the matched hardware, platform and browser profiles, so it is the compact form of everything else in this table. #} Device Id:{{ utils.get_human_readable(data.device, "deviceid") }} diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 6a223d00..8f364ac8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -48,6 +48,10 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=enterprise-data-file" + class MatchMetricsConsole(): def run(self, data_file, show_descs, logger, output): @@ -76,8 +80,9 @@ def run(self, data_file, show_descs, logger, output): # Uncomment BrowserName to include Browser component profile ID # in the device ID value. #restricted_properties=["ismobile", "hardwarename", "browsername"], - # If using the full on-premise data file this property will be - # present in the data file. See https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run + # Only the Enterprise data file has this property, and the free + # Lite data file leaves it out. Contact us for a copy at + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run # Only use the predictive graph to better handle variances # between the training data and the target User-Agent string. # For a more detailed description of the differences between @@ -85,8 +90,9 @@ def run(self, data_file, show_descs, logger, output): # https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run#DeviceDetection_Hash_DataSetProduction_Performance use_predictive_graph = True, use_performance_graph = False, - # We want to show the matching evidence characters as part of this example, so we have to set - # this flag to true. + # This example shows the matched evidence characters, which + # needs this option on. It is on by default, and is set here to + # show where to turn it off. update_matched_useragent = True).add_logger(logger).build() # The pipeline is closed at the end of the with block, which @@ -169,7 +175,9 @@ def get_component(property): if (show_descs == True): output(f" {propertyDescription}") - output() + output("") + if ExampleUtils.is_lite_data_file(pipeline.get_element("device")): + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) logger.log("info", "Finished Match Metrics Example") # Evidence values from a windows 11 device using a browser @@ -183,8 +191,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 5e605158..e3141e06 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -50,6 +50,10 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=enterprise-data-file" + bgRed = "\u001b[41m" fgWhite = "\u001b[37;1m" colReset = "\u001b[0m" @@ -84,6 +88,9 @@ def run(self, data_file, logger, output): self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) ExampleUtils.check_data_file(pipeline, logger) + if ExampleUtils.is_lite_data_file(pipeline.get_element("device")): + output("") + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) def outputEvidenceKeyDetails(self, engine, output): @@ -121,8 +128,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 500fdba7..c4b437f5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -59,6 +59,22 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME from ruamel.yaml import YAML +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=enterprise-data-file" + +# The properties only the Enterprise data file has, which are recorded as +# needing it when the data file in use does not have them. +ENTERPRISE_PROPERTIES = [ + "tac", + "devicetype", + "hardwarevendor", + "hardwarename", + "hardwaremodel", + "nativedevice", + "nativename", + "nativemodel"] + class OfflineProcessing(): def run(self, data_file, evidence_yaml, logger, output): """! @@ -68,6 +84,8 @@ def run(self, data_file, evidence_yaml, logger, output): @param evidence_yaml: File containing the yaml representation of the evidence to process @param logger: Logger to use within the pipeline @param output: Output file to write results to + @return: True when the data file lacks properties that only the + Enterprise data file has """ # In this example, we use the DeviceDetectionPipelineBuilder @@ -108,7 +126,12 @@ def run(self, data_file, evidence_yaml, logger, output): # and some evidence files contain them in user-agent strings. content = evidence_yaml.read().replace('\t', ' ') yaml_data = yaml.load_all(content) - + # The properties only the Enterprise data file has that the data + # file in use lacks, found once rather than for every record. + engine = pipeline.get_element("device") + missing = [name for name in ENTERPRISE_PROPERTIES + if not ExampleUtils.has_property(engine, name)] + try: # Keep going as long as we have more document records. for evidence in yaml_data: @@ -126,18 +149,25 @@ def run(self, data_file, evidence_yaml, logger, output): # Keys already have header. prefix in the YAML file headers[key] = str(evidence[key]) - self.analyseEvidence(headers, pipeline, output, yaml) + self.analyseEvidence( + headers, pipeline, output, yaml, missing) except BaseException as err: # We can't read the evidence values, so cant write them to the output. Will just # have to skip this entry. logger.log("error", err) + if missing: + # A comment keeps the output a valid YAML stream. + print("# " + ExampleUtils.get_enterprise_note(CONTACT_US), + file = output) + # write the yaml document end marker print("...", file = output) ExampleUtils.check_data_file(pipeline, logger) + return len(missing) > 0 - def analyseEvidence(self, evidence, pipeline, output, yaml): + def analyseEvidence(self, evidence, pipeline, output, yaml, missing): # FlowData is a data structure that is used to convey information required for # detection and the results of the detection through the pipeline. # Information required for detection is called "evidence" and usually consists @@ -165,14 +195,12 @@ def analyseEvidence(self, evidence, pipeline, output, yaml): values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") - values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") - values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") - values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") - values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") - values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") - values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") - values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") - values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") + # The value of a property the data file does not have is not + # read. + for property in ENTERPRISE_PROPERTIES: + values["device." + property] = ( + ExampleUtils.NEEDS_ENTERPRISE if property in missing + else ExampleUtils.get_human_readable(device, property)) # DeviceId is a unique identifier for the combination of hardware, operating # system, browser and crawler that has been detected. # Our device detection solution uses machine learning to find the optimal @@ -196,8 +224,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # This file contains the 20,000 most commonly seen combinations of header values # that are relevant to device detection. For example, User-Agent and UA-CH headers. @@ -212,9 +240,11 @@ def main(argv): if (data_file != None): with open(output_file, "w") as output: with open(evidence_file, "r") as input: - OfflineProcessing().run(data_file, input, logger, output) - logger.log("info", - f"Processing complete. See results in: '{output_file}'") + needs_enterprise = OfflineProcessing().run( + data_file, input, logger, output) + print(f"Processing complete. See results in: '{output_file}'") + if needs_enterprise: + print(ExampleUtils.get_enterprise_note(CONTACT_US)) else: logger.log("error", "Failed to find a device detection data file. Make sure the " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index d795b548..000186b4 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -51,6 +51,11 @@ from fiftyone_devicedetection_examples.example_utils import ExampleUtils # This example goes through a CSV of 20000 user agents and processes them, returning the time and information about the matches from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder +from fiftyone_devicedetection_shared.example_constants import UA_FILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-performance.py&utm_term=enterprise-data-file" # Here we make a function that processes a user agent # And returns if it is a mobile device @@ -150,11 +155,13 @@ def run(data_file, skip = False): if __name__ == "__main__": ap = argparse.ArgumentParser(description='Run detection benchmark.') ap.add_argument('-d', '--data_file', default='', help='Path to data file') - ap.add_argument('-u', '--user_agents_file', default='../../cxx/data/device-detection/20000 User Agents.csv', help='Path to user agents evidence file') + ap.add_argument('-u', '--user_agents_file', default='', help='Path to user agents evidence file') ap.add_argument('-j', '--json_output', default='', help='Output results in JSON format') args = ap.parse_args() if args.data_file == "": args.data_file = ExampleUtils.find_file("51Degrees-LiteV4.1.hash") + if args.user_agents_file == "": + args.user_agents_file = ExampleUtils.find_file(UA_FILE_NAME) # First we read the contents of the 20000 user agents file as a list with open(args.user_agents_file, newline='') as file: @@ -206,3 +213,16 @@ def run(data_file, skip = False): final_result["notmobile"] += result["notmobile"] print("Results", final_result) + + # Say what the Enterprise data file adds when the benchmark ran on the + # free Lite data file. The data file is opened with the low memory + # profile, which reads little of it. + with DeviceDetectionOnPremisePipelineBuilder( + data_file_path=args.data_file, + licence_keys="", + performance_profile='LowMemory', + add_javascript_builder=False, + usage_sharing=False, + auto_update=False).build() as check: + if ExampleUtils.is_lite_data_file(check.get_element("device")): + print(ExampleUtils.get_enterprise_offer(CONTACT_US)) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py index 60fe8f6d..015c35d3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py @@ -61,6 +61,11 @@ def stderr_print(*args, **kwargs): print(*args, **kwargs, file=sys.stderr, flush=True) +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-profile_metadata_console.py&utm_term=enterprise-data-file" + + class ProfileMetadataConsole(): # Properties we want to extract from each hardware profile # Note: TAC, HardwareName, NativeName, NativeModel are multi-value properties (string[]) @@ -199,9 +204,13 @@ def main(argv): if data_file is not None: ProfileMetadataConsole().run(data_file, output_file, logger, output_func, limit) else: - logger.log("error", - "Failed to find a device detection data file. " + - "Please provide the path to a TAC-HashV41.hash file as the first argument.") + # Only the Enterprise data file has the properties the example lists, + # so without one it says so and where to ask for one, and stops. + print("Listing the hardware profiles needs the TAC, HardwareVendor " + "and HardwareName properties, which only the Enterprise data " + "file has. Give the path to an Enterprise data file as the " + "first command line argument, or contact us for a copy at " + f"{CONTACT_US}") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html index a8c19cd5..31a88034 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html @@ -85,15 +85,15 @@

Detection results

- Hardware Vendor:{{ utils.get_human_readable(data.device, "hardwarevendor") }} - Hardware Name:{{ utils.get_human_readable(data.device, "hardwarename") }} - Device Type:{{ utils.get_human_readable(data.device, "devicetype") }} - Platform Vendor:{{ utils.get_human_readable(data.device, "platformvendor") }} - Platform Name:{{ utils.get_human_readable(data.device, "platformname") }} - Platform Version:{{ utils.get_human_readable(data.device, "platformversion") }} - Browser Vendor:{{ utils.get_human_readable(data.device, "browservendor") }} - Browser Name:{{ utils.get_human_readable(data.device, "browsername") }} - Browser Version:{{ utils.get_human_readable(data.device, "browserversion") }} + Hardware Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarevendor") }} + Hardware Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarename") }} + Device Type:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "devicetype") }} + Platform Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformvendor") }} + Platform Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformname") }} + Platform Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformversion") }} + Browser Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browservendor") }} + Browser Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browsername") }} + Browser Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browserversion") }} diff --git a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py index 79bf78c5..f5e10453 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py @@ -35,6 +35,13 @@ from fiftyone_devicedetection_examples.cloud.gettingstarted_console import GettingStartedConsole from fiftyone_devicedetection_examples.cloud.configurator_console import ConfiguratorConsole +from .live_cloud import find_resource_key, no_resource_key + +NO_KEY = no_resource_key( + "the device product, and the hardware product for the lookups", + "device detection properties, and for the hardware profiles the TAC " + "and native model lookups return", several=True) + # Text that shows an example printed a programming fault rather than a # result. An example that reaches any of these is broken, however little of # the data the resource key is entitled to. @@ -50,17 +57,12 @@ class DeviceDetectionExampleTests(unittest.TestCase): - # Init method - specify a resource key here, or set one in the - # environment variable named by ExampleUtils.RESOURCE_KEY_ENV_VAR. def setUp(self): - self.resource_key = ExampleUtils.get_resource_key() + self.resource_key = find_resource_key() self.logger = Logger() if not self.resource_key: - # Skipping rather than passing, so a run without a key is not - # mistaken for a run that proved something. The message names - # the variable that was wanted. - self.skipTest(ExampleUtils.get_missing_resource_key_message()) + self.fail(NO_KEY) def run_example(self, example): """! diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py new file mode 100644 index 00000000..c035294d --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py @@ -0,0 +1,140 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Runs the on-premise examples as they run from a fresh clone, with no +# arguments and no licence key, so that they use the free Lite data file. +# Each must finish, show the properties only the Enterprise data file has +# as needing it, and give the link to ask for a copy. + +import io +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_pipeline_core.logger import Logger + +from fiftyone_devicedetection_examples.example_utils import ExampleUtils +from fiftyone_devicedetection_examples.onpremise import csv_profile_metadata +from fiftyone_devicedetection_examples.onpremise import datafileupdate_console +from fiftyone_devicedetection_examples.onpremise import gettingstarted_console +from fiftyone_devicedetection_examples.onpremise import match_metrics +from fiftyone_devicedetection_examples.onpremise import metadata_console +from fiftyone_devicedetection_examples.onpremise import offlineprocessing +from fiftyone_devicedetection_examples.onpremise import profile_metadata_console +from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME +from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME + +# The settings the examples read, which a fresh clone does not have. +SETTINGS = ["license_key", "LICENSE_KEY"] + +# The evidence records the offline processing test reads. The example's own +# file has 20,000, which take minutes. +OFFLINE_RECORDS = 20 + + +def run_without_settings(main, argv=None): + """Runs an example's main with the settings taken out of the + environment, and returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + main(argv or []) + return printed.getvalue() + + +def first_records(path, count): + """The first records of a YAML evidence file.""" + lines = [] + with open(path, encoding="utf-8") as evidence: + for line in evidence: + if line.rstrip() == "---": + count -= 1 + if count < 0: + break + lines.append(line) + return io.StringIO("".join(lines)) + + +class ExamplesWithoutSettingsTests(unittest.TestCase): + + def test_gettingstarted_console(self): + printed = run_without_settings(gettingstarted_console.main) + self.assertIn( + "Hardware Vendor: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), + printed) + + def test_match_metrics(self): + printed = run_without_settings(match_metrics.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(match_metrics.CONTACT_US), + printed) + + def test_metadata_console(self): + printed = run_without_settings(metadata_console.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), + printed) + + def test_offline_processing(self): + output = io.StringIO() + lacks_properties = offlineprocessing.OfflineProcessing().run( + ExampleUtils.find_file(LITE_DATAFILE_NAME), + first_records( + ExampleUtils.find_file(EVIDENCE_FILE_NAME), OFFLINE_RECORDS), + Logger(min_level="error"), + output) + self.assertTrue(lacks_properties) + self.assertIn( + "device.hardwarevendor: " + ExampleUtils.NEEDS_ENTERPRISE, + output.getvalue()) + self.assertIn( + "# " + ExampleUtils.get_enterprise_note( + offlineprocessing.CONTACT_US), + output.getvalue()) + + def test_datafileupdate_console(self): + printed = run_without_settings(datafileupdate_console.main) + self.assertIn("The data file in use is the 'Lite' file", printed) + self.assertIn(datafileupdate_console.CONTACT_US, printed) + + # The two examples below need files that are not free, so in a fresh + # clone the search for them finds nothing, which the tests stand in for. + + def test_csv_profile_metadata(self): + with mock.patch.object(ExampleUtils, "find_file", return_value=None): + printed = run_without_settings(csv_profile_metadata.main) + self.assertIn(csv_profile_metadata.CSV_FILE_NAME, printed) + self.assertIn(csv_profile_metadata.CONTACT_US, printed) + + def test_profile_metadata_console(self): + with mock.patch.object(ExampleUtils, "find_file", return_value=None): + printed = run_without_settings(profile_metadata_console.main) + self.assertIn("only the Enterprise data file has", printed) + self.assertIn(profile_metadata_console.CONTACT_US, printed) + + +if __name__ == "__main__": + unittest.main() diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 0b6222c0..b0932b5a 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -33,6 +33,7 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key run_performance_tests diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 632751ae..5f3d168c 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -61,8 +61,8 @@ def __init__( polling_interval=30, update_time_maximum_randomisation=10, create_temp_data_copy=True, - update_matched_useragent=False, - performance_profile="LowMemory", + update_matched_useragent=None, + performance_profile="Balanced", reuse_temp_file=False, concurrency=multiprocessing.cpu_count(), update_on_start=False, @@ -90,10 +90,20 @@ def __init__( @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -101,7 +111,9 @@ def __init__( @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int @@ -209,7 +221,10 @@ def __init__( config.setUseTempFile(True) config.setReuseTempFile(reuse_temp_file) - config.setUpdateMatchedUserAgent(update_matched_useragent) + + # Left unset, the native default applies, which is on. + if update_matched_useragent is not None: + config.setUpdateMatchedUserAgent(update_matched_useragent) if allow_unmatched == True or allow_unmatched == False: config.setAllowUnmatched(allow_unmatched) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py index 133b9f2e..28ce1b70 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py @@ -80,10 +80,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -91,7 +101,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py new file mode 100644 index 00000000..a40f04de --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py @@ -0,0 +1,115 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule import ConfigHashSwig +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import DeviceDetectionOnPremise +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +# The free Lite data file, where cxx/data/get-data-files.ps1 and +# get-data-files.sh download it and where CI links it in. +DATA_FILE = os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "cxx", + "data", "device-detection", "51Degrees-LiteV4.1.hash")) + +# The collections whose settings the configuration exposes. +COLLECTIONS = ( + "Strings", "Properties", "Values", "Profiles", "Nodes", "ProfileOffsets") + +# Each profile name the engine accepts and the native setter it maps to. +PROFILES = { + "LowMemory": "setLowMemory", + "MaxPerformance": "setMaxPerformance", + "Balanced": "setBalanced", + "BalancedTemp": "setBalancedTemp", + "HighPerformance": "setHighPerformance", +} + + +def collection_settings(config): + """ + Whether each collection is loaded into memory and how many items it + caches, which are what a performance profile sets. + """ + settings = {} + for name in COLLECTIONS: + collection = getattr(config, "get" + name)() + settings[name] = (collection.getLoaded(), collection.getCapacity()) + return settings + + +def profile_settings(setter): + """The collection settings the native code gives a profile.""" + config = ConfigHashSwig() + getattr(config, setter)() + return collection_settings(config) + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free Lite data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class PerformanceProfileDefaultTests(unittest.TestCase): + """ + The engine keeps no name for the profile in use, so these tests compare + the collection settings of the configuration the engine gives the + native code with the settings each profile has there. MaxPerformance + and LowMemory have the same collection settings, and Balanced and + BalancedTemp do too, so neither pair can be told apart this way. + """ + + def _engine_settings(self, **settings): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", **settings) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + self.assertIsNotNone(engine.engine) + return collection_settings(engine.config) + + def test_balanced_and_low_memory_differ(self): + self.assertNotEqual( + profile_settings("setBalanced"), profile_settings("setLowMemory")) + + def test_engine_defaults_use_balanced(self): + self.assertEqual( + profile_settings("setBalanced"), self._engine_settings()) + + def test_builder_defaults_use_balanced(self): + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, + licence_keys="").build() + self.addCleanup(pipeline.close) + engine = pipeline.get_element("device") + self.assertEqual( + profile_settings("setBalanced"), + collection_settings(engine.config)) + + def test_each_named_profile_is_used(self): + for name, setter in PROFILES.items(): + with self.subTest(name): + self.assertEqual( + profile_settings(setter), + self._engine_settings(performance_profile=name)) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py new file mode 100644 index 00000000..155053c5 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py @@ -0,0 +1,89 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +# The free Lite data file, where cxx/data/get-data-files.ps1 and +# get-data-files.sh download it and where CI links it in. +DATA_FILE = os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "cxx", + "data", "device-detection", "51Degrees-LiteV4.1.hash")) + +MOBILE_UA = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) " + "AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114") + +# The character the native code writes where the evidence was not matched. +UNMATCHED = "_" + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free Lite data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class UpdateMatchedUserAgentTests(unittest.TestCase): + """ + The Update Matched User Agent option is on unless it is turned off, so + the UserAgents property holds the characters of the evidence that were + matched. + """ + + def _user_agents(self, **settings): + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, licence_keys="", + **settings).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("header.user-agent", MOBILE_UA) + flowdata.process() + return flowdata.device.useragents.value() + + def assert_matched_from_the_evidence(self, user_agents): + """ + Each matched User-Agent has the evidence's characters where they were + matched, and at least one character was matched. + """ + self.assertGreaterEqual(len(user_agents), 1) + matched = 0 + for user_agent in user_agents: + for index, character in enumerate(user_agent): + if character != UNMATCHED: + self.assertEqual(MOBILE_UA[index], character, user_agent) + matched += 1 + self.assertGreater(matched, 0, user_agents) + + def test_builder_defaults_populate_user_agents(self): + self.assert_matched_from_the_evidence(self._user_agents()) + + def test_turned_on_populates_user_agents(self): + self.assert_matched_from_the_evidence( + self._user_agents(update_matched_useragent=True)) + + def test_turned_off_leaves_user_agents_empty(self): + user_agents = self._user_agents(update_matched_useragent=False) + self.assertGreaterEqual(len(user_agents), 1) + self.assertEqual([""] * len(user_agents), user_agents) diff --git a/device-detection/fiftyone_devicedetection_shared/setup.py b/device-detection/fiftyone_devicedetection_shared/setup.py index 07b89cdf..c57092a3 100644 --- a/device-detection/fiftyone_devicedetection_shared/setup.py +++ b/device-detection/fiftyone_devicedetection_shared/setup.py @@ -46,7 +46,20 @@ def read(file_name): "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=documentation", "Source": "https://github.com/51Degrees/51degrees-python", }, - description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). This project contains 51Degrees Device Detection engines that can be used with the Pipeline API."), + description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. This package holds the code shared by the cloud and on-premise engines for the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL."), + keywords=[ + "51degrees", + "device-detection", + "user-agent", + "user-agent-parser", + "client-hints", + "browser-detection", + "mobile-detection", + "crawler-detection", + "wurfl", + "deviceatlas", + "uaparser" + ], long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.10", diff --git a/ip-intelligence/ci/fetch-assets.ps1 b/ip-intelligence/ci/fetch-assets.ps1 index a26332bc..f4653aa8 100644 --- a/ip-intelligence/ci/fetch-assets.ps1 +++ b/ip-intelligence/ci/fetch-assets.ps1 @@ -31,6 +31,11 @@ foreach ($asset in $assets) { # The ASN file is not an asset, so the data repository's script downloads it # beside its link in cxx/data/ip-intelligence, where the tests look for it. +# The examples look for the Lite data file in the data repository, under the +# name its script saves it as, so the asset is linked there too rather than +# downloaded again. +New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$PSScriptRoot/../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" + $global:LASTEXITCODE = 0 & "$PSScriptRoot/../../cxx/data/get-data-files.ps1" 51Degrees-IPIV4AsnIpiV41.ipi if ($LASTEXITCODE -ne 0) { diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index 5e50e871..f57e296c 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -14,17 +14,18 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so cloud integration tests will not run." -} -else { - # Says what that key is entitled to, so a run with no IP data is read - # as an entitlement gap rather than as broken code. Prints product - # names and tiers only, never the key. Found through this script's own - # folder, because common-ci runs it from the common-ci checkout, where - # './ci/report-key-products.ps1' does not exist. - & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } +# Says what that key is entitled to, so a run with no IP data is read as an +# entitlement gap rather than as broken code. Prints product names and tiers +# only, never the key. Found through this script's own folder, because +# common-ci runs it from the common-ci checkout, where +# './ci/report-key-products.ps1' does not exist. +& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey + # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get # installed. That's why we install it here, as a special case. diff --git a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py index a2511324..0450b9eb 100644 --- a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py +++ b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py @@ -24,7 +24,7 @@ # First create the IP intelligence pipeline with the desired settings. -data_file = "../assets/51Degrees-LiteIpiV41.ipi" +data_file = "../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" resource_key = "!!YOUR_RESOURCE_KEY!!" # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to diff --git a/ip-intelligence/fiftyone_ipintelligence/readme.md b/ip-intelligence/fiftyone_ipintelligence/readme.md index 1b1c9eec..4b546d11 100644 --- a/ip-intelligence/fiftyone_ipintelligence/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence/readme.md @@ -25,7 +25,7 @@ pipeline = IpIntelligencePipelineBuilder(resource_key="YOUR_RESOURCE_KEY").build # On-premise pipeline = IpIntelligencePipelineBuilder( - data_file_path="51Degrees-LiteIpiV41.ipi", + data_file_path="51Degrees-LiteV41.ipi", licence_keys="", usage_sharing=False).build() diff --git a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py index 8576e4cf..e17ee5c5 100644 --- a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py @@ -82,7 +82,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini index 4aa63411..668932c9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini @@ -12,7 +12,13 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. -e../fiftyone_ipintelligence_shared/ + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_cloud {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index c41c0630..dd0f3423 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -33,23 +33,32 @@ python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_web ## On-premise examples -Require a `.ipi` data file in the format the engine reads (see +Use a `.ipi` data file in the format the engine reads (see [Data files](../readme.md#data-files)). The data repository, at -`cxx/data`, links to a freely available network-only file -(`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to, and an -example downloads it when it is missing. It only contains the ASN/network -properties, so location and registration properties are reported as "not -present in the data file". - -For the full set of properties (location, registration, etc.), a more -complete data file is required - to obtain one for testing please -[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=on-premise-examples) - and supply its path as -the first command line argument. +`cxx/data`, links to the two free data files, and an example downloads the +one it uses the first time it is missing, so the examples run from a fresh +clone with no settings. The free Lite data file (`51Degrees-LiteV41.ipi`) +has the registration properties, such as RegisteredName and +RegisteredCountry, and most examples use it. The free ASN data file +(`51Degrees-IPIV4AsnIpiV41.ipi`) has the ASN properties and is far smaller, +so the performance and data file update examples use it. + +The location, range and connection properties are only in the Enterprise +data file. An example run on a free data file shows each property it lacks +as needing the Enterprise data file, and gives a link to +[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=on-premise-examples) about a copy. +The countries and comparison examples need those properties throughout, so +on a free data file they say which they need and stop. Supply the path to +an Enterprise data file as the first command line argument, or with `-d`, +to see every property. - `onpremise/gettingstarted_console.py` - resolve IP addresses from the console using a local data file. - `onpremise/metadata_console.py` - list the properties available from the data file. +- `onpremise/countries_console.py` - list the countries an IP address is + most likely in, with their names translated (needs the Enterprise data + file). - `onpremise/offlineprocessing.py` - batch process the evidence file from the data repository. - `onpremise/performance_console.py` - benchmark the number of IP lookups @@ -57,7 +66,7 @@ the first command line argument. - `onpremise/compare_console.py` - compare the location returned for each IP address against a CSV of known-true locations. - `onpremise/datafileupdate_console.py` - demonstrate automatic data file - updates (requires a license key). + updates (needs a licence key, and without one says where to ask for one). - `onpremise/mixed_console.py` - combine device detection and IP intelligence in a single on-premise pipeline (requires the device detection package and a `.hash` data file). @@ -67,10 +76,11 @@ the first command line argument. ``` python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console [data-file] python -m fiftyone_ipintelligence_examples.onpremise.metadata_console [data-file] +python -m fiftyone_ipintelligence_examples.onpremise.countries_console [data-file] python -m fiftyone_ipintelligence_examples.onpremise.offlineprocessing [data-file [evidence-file]] python -m fiftyone_ipintelligence_examples.onpremise.performance_console [-d data-file] [-e evidence-file] python -m fiftyone_ipintelligence_examples.onpremise.compare_console [-d data-file] [-t truth-file] -python -m fiftyone_ipintelligence_examples.onpremise.datafileupdate_console [data-file] +python -m fiftyone_ipintelligence_examples.onpremise.datafileupdate_console [license-key [data-file]] python -m fiftyone_ipintelligence_examples.onpremise.mixed_console [hash-file [ipi-file]] python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_web ``` diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index 81eb5829..3ffd5499 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -23,6 +23,7 @@ from datetime import datetime, timedelta import os import shlex +import shutil import subprocess from pathlib import Path @@ -50,10 +51,21 @@ class ExampleUtils: # belongs in this folder, beside its .link file. DATA_FOLDER = os.path.join("cxx", "data", "ip-intelligence") + # The data repository's device detection folder, for the examples that + # use both products. + DEVICE_DETECTION_DATA_FOLDER = os.path.join("cxx", "data", "device-detection") + # Sent with each download, so the downloads these examples make can be - # counted. + # counted. It is left out inside GitHub Actions, where the data + # repository's script names the build instead, so that test runs are + # not counted as people using the examples. DOWNLOAD_SOURCE = "examples-51degrees-python" + # Shown in place of a value for a property that the data file in use + # does not have. Every property the examples ask for is in the + # Enterprise data file. + NEEDS_ENTERPRISE = "needs the Enterprise data file" + @staticmethod def get_human_readable(ip_data, property): try: @@ -124,7 +136,7 @@ def find_file(file_name): Find the specified file. First checks the data file environment variable, then searches up the directory tree from the current working directory looking in 'assets' and the data repository's - folder. A file the data repository links to is downloaded there + folders. A file the data repository links to is downloaded there when it is missing. """ @@ -134,17 +146,20 @@ def find_file(file_name): current = Path(os.getcwd()).resolve() roots = [current, *current.parents] + folders = [ + ExampleUtils.DATA_FOLDER, + ExampleUtils.DEVICE_DETECTION_DATA_FOLDER] for root in roots[:8]: - for sub_dir in [".", "assets", ExampleUtils.DATA_FOLDER]: + for sub_dir in [".", "assets", *folders]: candidate = os.path.join(str(root), sub_dir, file_name) if os.path.isfile(candidate): return candidate for root in roots[:8]: - candidate = os.path.join( - str(root), ExampleUtils.DATA_FOLDER, file_name) - if os.path.isfile(candidate + ".link"): - return ExampleUtils.__download(candidate) + for folder in folders: + candidate = os.path.join(str(root), folder, file_name) + if os.path.isfile(candidate + ".link"): + return ExampleUtils.__download(candidate) return None @@ -160,23 +175,34 @@ def __download(path): name = os.path.basename(path) data = os.path.dirname(os.path.dirname(path)) + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + environment = None if os.name == "nt": + # PowerShell 7 when it is on the path, otherwise Windows + # PowerShell, which, started from PowerShell 7, inherits its + # module path and then cannot load its own commands. + power_shell = "pwsh" if shutil.which("pwsh") else "powershell" command = [ - "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", - "-File", os.path.join(data, "get-data-files.ps1"), name, - "-Source", ExampleUtils.DOWNLOAD_SOURCE] + power_shell, "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name] + if not in_actions: + command += ["-Source", ExampleUtils.DOWNLOAD_SOURCE] + if power_shell == "powershell": + environment = dict(os.environ) + environment.pop("PSModulePath", None) text = subprocess.list2cmdline(command) else: - command = [ - "sh", os.path.join(data, "get-data-files.sh"), - "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + command = ["sh", os.path.join(data, "get-data-files.sh")] + if not in_actions: + command += ["-s", ExampleUtils.DOWNLOAD_SOURCE] + command.append(name) text = shlex.join(command) # Flushed first, because the script writes to the same output. print(f"{name} is missing, so it is being downloaded by running\n" f"{text}", flush=True) try: - subprocess.run(command, check=False) + subprocess.run(command, check=False, env=environment) except OSError: pass if os.path.isfile(path): @@ -185,6 +211,132 @@ def __download(path): f"again.\n{text}") return None + @staticmethod + def has_property(engine, name): + + """! + True when the engine's data file has the property. The free Lite data + file has the registration properties, such as RegisteredName, and the + free ASN data file the ASN properties, such as AsnName. The data + file's own list is read, because the engine's properties are only + those it was built to give. + """ + + properties = engine.engine.getMetaData().getProperties() + name = name.lower() + for index in range(properties.getSize()): + if properties.getByIndex(index).getName().lower() == name: + return True + return False + + @staticmethod + def is_free_data_file(engine): + + """! + True when the engine uses one of the free data files, Lite or ASN. + Only the Enterprise data file has the range properties, so a check + for IpRangeStart tells them apart. + """ + + return not ExampleUtils.has_property(engine, "IpRangeStart") + + @staticmethod + def is_free_data_file_at(data_file): + + """! + True when the data file at the path is one of the free data files. + The data file is opened with the low memory profile, which reads + little of it. + """ + + with ExampleUtils.__open(data_file) as pipeline: + return ExampleUtils.is_free_data_file( + pipeline.flow_elements_list["ip"]) + + @staticmethod + def get_enterprise_value(engine, ip_data, property): + + """! + Format a property for display, or say that it needs the Enterprise + data file when the engine's data file does not have it. The value + is only read when the data file has the property. + """ + + if not ExampleUtils.has_property(engine, property): + return ExampleUtils.NEEDS_ENTERPRISE + return ExampleUtils.get_human_readable(ip_data, property) + + @staticmethod + def lacks_enterprise_properties(data_file, purpose, properties, output, + contact_us): + + """! + For an example that needs properties only the Enterprise data file + has, report when the data file has none of them, and then say so and + where to ask for a copy of the Enterprise data file. The data file is + opened with the low memory profile, which reads little of it. + @param purpose: What the example does, as the start of a sentence, + for example "Comparing known locations with IP Intelligence". + @param contact_us: The link to the contact page, tagged with the + example's file, so that an enquiry can be traced to the example. + """ + + with ExampleUtils.__open(data_file) as pipeline: + engine = pipeline.flow_elements_list["ip"] + if any(ExampleUtils.has_property(engine, name) + for name in properties): + return False + if len(properties) == 1: + names = properties[0] + else: + names = ", ".join(properties[:-1]) + " and " + properties[-1] + output(f"{purpose} needs the {names} properties, which only the " + "Enterprise data file has. The data file in use has none of " + "them, so the example stops here. Contact us for a copy of " + f"the Enterprise data file at {contact_us}") + return True + + @staticmethod + def get_enterprise_note(contact_us): + + """! + Where to ask for a copy of the Enterprise data file, written after an + example has shown properties that the data file in use does not + have. The link is tagged with the example's file, so that an enquiry + can be traced to the example. + """ + + return ("Some of these properties need the Enterprise data file. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def get_enterprise_offer(contact_us): + + """! + What the Enterprise data file adds and where to ask for a copy, + written after an example that shows no location properties has run + on a free data file. + """ + + return ("This example used a free data file. The Enterprise data " + "file adds the location, range and connection properties. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def __open(data_file): + + """! + A pipeline that reads the data file with the low memory profile. + """ + + from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder + return IpIntelligencePipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + @staticmethod def get_data_file_date(pipeline): engine = pipeline.flow_elements_list["ip"] @@ -194,8 +346,9 @@ def get_data_file_date(pipeline): def check_data_file(pipeline, logger): """! - Display information about the data file and log warnings if it - is old or a Lite file. + Display information about the data file and log a warning if it is + old. An example run on a free data file says what the Enterprise + data file adds in its own output. """ engine = pipeline.flow_elements_list["ip"] @@ -223,13 +376,6 @@ def check_data_file(pipeline, logger): f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " f"on Windows.") - if product == "Lite": - logger.log("warning", - "This example is using the 'Lite' data file. This is " - "used for illustration, and has limited properties. A " - "more complete data file can be obtained from " - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-example_utils.py&utm_term=lite-data-file") - @staticmethod def __get_env_variable(name): value = os.environ.get(name) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index e418f786..d928f86f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -39,9 +39,10 @@ # true location and the returned location is then calculated, along with the # location confidence and whether an area was returned. # -# This example requires an enterprise IP intelligence data file (.ipi). To -# obtain an enterprise data file for testing, please contact us: -# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=top +# This example needs the location properties only the Enterprise IP +# intelligence data file has. With the free Lite data file, which it uses by +# default, it says which and stops. To obtain an Enterprise data file, please +# contact us: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=top # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -54,11 +55,19 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME # Name of the truth CSV file in the data repository. GEOIP_COMPARISON_EVIDENCE_FILE_NAME = "geoip_comparison_evidence.csv" +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=enterprise-data-file" + +# The properties the comparison reads, which only the Enterprise data file +# has. +LOCATION_PROPERTIES = ["Latitude", "Longitude", "LocationConfidence", "Country"] + # Mean radius of the earth in kilometres, used for the distance calculation. EARTH_RADIUS_KM = 6371.0 @@ -175,12 +184,19 @@ def main(argv): help="Maximum number of truth records to process") args = ap.parse_args(argv) - data_file = args.data_file or ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + data_file = args.data_file or ExampleUtils.find_file(LITE_DATAFILE_NAME) truth_file = args.truth_file or ExampleUtils.find_file( GEOIP_COMPARISON_EVIDENCE_FILE_NAME) logger = Logger(min_level="info") + if data_file and ExampleUtils.lacks_enterprise_properties( + data_file, + "Comparing known locations with IP Intelligence", + LOCATION_PROPERTIES, + print, + CONTACT_US): + return if data_file and truth_file: with open(args.output_file, "w", newline="", encoding="utf-8") as fout: CompareConsole().run( diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index eb239813..6e33ef2b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -32,10 +32,10 @@ # 2. How to supply the browser language so the names are localized # 3. How to read the ordered "All" lists used to build a country dropdown # -# This example uses a local IP intelligence data file (.ipi). A data file that -# contains country information is needed to see weighted countries lead the -# list; with the network-only ASN data file the lists are simply every known -# country in alphabetical order. +# The country codes this example orders and translates are only in the +# Enterprise data file. With the free Lite data file, which it uses by +# default, the example says so and stops. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-countries_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -49,7 +49,15 @@ add_country_translation) from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_VALUES -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-countries_console.py&utm_term=enterprise-data-file" + +# The properties the country lists are built from, which only the Enterprise +# data file has. +COUNTRY_PROPERTIES = ["CountryCodesGeographical", "CountryCodesPopulation"] # The browser language to translate the country names to. A value such as # "fr_FR" or a full Accept-Language header is accepted; leave blank to keep the @@ -146,12 +154,18 @@ def main(argv): data_file = ( argv[0] if len(argv) > 0 - else ExampleUtils.find_file(ASN_DATAFILE_NAME)) + else ExampleUtils.find_file(LITE_DATAFILE_NAME)) logger = Logger(min_level="info") if data_file: - CountriesConsole().run(data_file, logger, print) + if not ExampleUtils.lacks_enterprise_properties( + data_file, + "Listing the countries an IP address is most likely in", + COUNTRY_PROPERTIES, + print, + CONTACT_US): + CountriesConsole().run(data_file, logger, print) else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 37200afd..54971693 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -32,10 +32,11 @@ # - Daily auto-update # # ## License Key -# In order to run this example you will need a 51Degrees Enterprise license, -# which can be purchased from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=top. -# Supply the license key as the first command line argument or in the -# 'license_key' environment variable. +# Updating the data file needs a 51Degrees licence key for the Enterprise +# data file. Without one, the example shows the free ASN data file and says +# where to ask for a key. Supply the licence key as the first command line +# argument or in the 'license_key' environment variable. Contact us for a key: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=header # # ## Update on start-up # Configure the pipeline to download an Enterprise data file on start-up. @@ -69,6 +70,7 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus @@ -76,6 +78,10 @@ # a command line argument. UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +# Where to ask for a licence key and a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" + # The product type passed to the 51Degrees distributor. This identifies the # IP intelligence Enterprise product and may need to be adjusted to match # the product your license entitles you to. @@ -100,6 +106,33 @@ def clear(self): class DataFileUpdateConsole(): + def explain_license_key(self, data_file, output): + # Without a licence key the example cannot download a data file, so + # it shows the data file it would start from, the free ASN file + # unless another is given, and says what an update needs. + if not data_file: + data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) + if data_file and os.path.isfile(data_file): + pipeline = IpIntelligencePipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + with pipeline: + engine = pipeline.flow_elements_list["ip"].engine + published = engine.getPublishedTime() + date = datetime(published.getYear(), published.getMonth(), + published.getDay()) + output(f"The data file in use is the '{engine.getProduct()}' " + f"file published on {date:%Y-%m-%d}, at '{data_file}'.") + output("") + output("Updating the data file downloads the Enterprise data file, " + "which needs a licence key. Give the key as the first command " + "line argument, or in the environment variable " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key " + f"and a copy at {CONTACT_US}") + def run(self, data_file, license_key, interactive, logger, output): logger.log("info", "Starting example") @@ -108,14 +141,8 @@ def run(self, data_file, license_key, interactive, logger, output): license_key = os.environ.get(UPDATE_EXAMPLE_LICENSE_KEY_NAME) if not license_key: - logger.log("error", - "In order to run this example you will need a 51Degrees " - "Enterprise license, which can be obtained on a trial basis " - "or purchased from our pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=run. " - "You must supply the license key as an argument to this " - "program, or in an environment variable named " - f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'") - raise Exception("No license key available") + self.explain_license_key(data_file, output) + return # Work out where the downloaded file will be put. The directory must # exist; the file itself need not. @@ -161,6 +188,9 @@ def run(self, data_file, license_key, interactive, logger, output): # is configured to use a temporary copy. data_file_path = data_file, create_temp_data_copy = True, + # The Enterprise data file is large, so the LowMemory profile + # reads it from disk rather than holding parts of it in memory. + performance_profile = "LowMemory", # The update listener notifies us when an update is complete. data_file_update_service = update_service, # A license key is required for automatic updates. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 4988da7d..70c2e121 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -32,10 +32,12 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example uses a local IP intelligence data file (.ipi). The data -# repository links to a freely available data file, which the example -# downloads when it is missing. A more complete data file can be supplied -# as the first command line argument. +# By default the example uses the free Lite IP intelligence data file, which +# it downloads when it is missing. The Lite data file has the registration +# properties, and the example shows each property only the Enterprise data +# file has as needing it. An Enterprise data file can be supplied as the first +# command line argument. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -45,12 +47,15 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_VALUES -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=enterprise-data-file" # The properties displayed for each IP address. These mirror the .NET -# getting started example. Properties that are not present in the data file -# being used (for example the network-only ASN data file) are reported as -# such rather than causing an error. +# getting started example. A property the data file in use does not have is +# shown as needing the Enterprise data file. OUTPUT_PROPERTIES = [ "RegisteredName", "RegisteredOwner", @@ -99,8 +104,12 @@ def run(self, data_file, logger, output): ExampleUtils.check_data_file(pipeline, logger) # carry out some sample lookups + needs_enterprise = False for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + needs_enterprise |= self.analyseEvidence( + values, pipeline, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) def analyseEvidence(self, evidence, pipeline, output): @@ -141,11 +150,16 @@ def analyseEvidence(self, evidence, pipeline, output): # Display the results of the lookup. See the property # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence # for details of all available properties. + # The engine says which properties its data file has. + engine = pipeline.flow_elements_list["ip"] + needs_enterprise = False for name in OUTPUT_PROPERTIES: - self.outputValue(ip_data, name, message) + needs_enterprise |= self.outputValue( + engine, ip_data, name, message) output("".join(message)) + return needs_enterprise - def outputValue(self, ip_data, name, message): + def outputValue(self, engine, ip_data, name, message): # Individual result values have a wrapper called # `AspectPropertyValue`. This functions similarly to # a null-able type. @@ -153,22 +167,23 @@ def outputValue(self, ip_data, name, message): # `value` method will throw an exception. # `AspectPropertyValue` also includes the `no_value_message` # method, which describes why the value has not been set. - # `get_value` returns None when the property is not present in the - # data file at all (for example the network-only ASN data file), - # rather than raising. - value = ExampleUtils.get_value(ip_data, name.lower()) - if value is None: - message.append(f"\t{name}: This property is not present in the data file.\n") - elif value.has_value(): + # A property the data file does not have needs the Enterprise data + # file, and its value is not read. + if not ExampleUtils.has_property(engine, name): + message.append(f"\t{name}: {ExampleUtils.NEEDS_ENTERPRISE}\n") + return True + value = ip_data.get(name.lower()) + if value.has_value(): message.append(f"\t{name}: {ExampleUtils.format_value(value.value())}\n") else: message.append(f"\t{name}: {value.no_value_message()}\n") + return False def main(argv): # Use the supplied path for the data file or find the data file in the # project space. - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ASN_DATAFILE_NAME) + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. logger = Logger(min_level="info") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index 56ea9799..e6f25806 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -44,6 +44,10 @@ add_country_translation, build_country_options) from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_web-app.py&utm_term=enterprise-data-file" + app = Flask(__name__) pipeline = None @@ -69,6 +73,12 @@ def get_value(ip_data, property): def country_dropdown(translated): + # Only the Enterprise data file has the country codes the dropdown is + # built from. + if not ExampleUtils.has_property( + pipeline.flow_elements_list["ip"], "CountryCodesGeographical"): + return (f"

The country dropdown {ExampleUtils.NEEDS_ENTERPRISE}. " + f"Contact us for a copy.

") # The geographical "All" lists are index-aligned: option i is # (CountryCodesGeographicalAll[i], CountryNamesGeographicalAllTranslated[i]), diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index 3819776c..f9e4a0fb 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -25,7 +25,8 @@ # # This example shows how to list the properties available from the # on-premise IP intelligence engine, along with their types and -# descriptions. +# descriptions. By default it uses the free Lite data file, which it +# downloads when it is missing. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -34,7 +35,11 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=enterprise-data-file" class MetadataConsole(): @@ -59,9 +64,13 @@ def run(self, data_file, logger, output): if property_metadata.get("description"): output(f" {property_metadata['description']}") + if ExampleUtils.is_free_data_file(engine): + output("") + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) + def main(argv): - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) logger = Logger(min_level="info") @@ -69,10 +78,11 @@ def main(argv): MetadataConsole().run(data_file, logger, print) else: logger.log("error", - "Failed to find an IP intelligence data file. Supply its path " - "as the first command line argument. To obtain an enterprise " - "data file for testing, please contact us: " - "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=data-file-required") + "Failed to find an IP intelligence data file. Make sure the " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " + "data file as the first command line argument.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 2f4f5e18..60fda04e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -28,11 +28,15 @@ # User-Agent (used by device detection) and an IP address (used by IP # intelligence), and both engines process the flow data. # -# Two data files are required: +# Two data files are used, which may be supplied as the first two command +# line arguments: # - a Device Detection data file (.hash) # - an IP Intelligence data file (.ipi) # -# They may be supplied as the first two command line arguments. +# By default the free Lite data files are used, which are downloaded when +# they are missing, and each property only the Enterprise data files have is +# shown as needing them. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-mixed_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -46,7 +50,11 @@ from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import IpIntelligenceOnPremise from fiftyone_ipintelligence_examples.example_utils import ExampleUtils -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data files, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-mixed_console.py&utm_term=enterprise-data-file" # Default name of the device detection data file (the lite file the data # repository links to). @@ -143,10 +151,15 @@ def run(self, device_file, ip_file, logger, output): # The pipeline is closed at the end of the with block, which closes # both engines and releases the native data they hold. with pipeline: + needs_enterprise = False for evidence in COMBINED_EVIDENCE: - self.analyse_evidence(evidence, pipeline, output) + needs_enterprise |= self.analyse_evidence( + evidence, pipeline, device_engine, ip_engine, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) - def analyse_evidence(self, evidence, pipeline, output): + def analyse_evidence(self, evidence, pipeline, device_engine, ip_engine, + output): # The FlowData is closed at the end of the with block, which releases # the results of both engines straight away. @@ -165,36 +178,42 @@ def analyse_evidence(self, evidence, pipeline, output): data.process() message = [] + needs_enterprise = False # Device detection results, keyed under "device". message.append("Device Detection Results:\n") for name in DEVICE_PROPERTIES: - self.outputValue(data.device, name, message) + needs_enterprise |= self.outputValue( + device_engine, data.device, name, message) # IP intelligence results, keyed under "ip". message.append("\nIP Intelligence Results:\n") for name in IP_PROPERTIES: - self.outputValue(data.ip, name, message) + needs_enterprise |= self.outputValue( + ip_engine, data.ip, name, message) output("".join(message)) - - def outputValue(self, aspect_data, name, message): - # `get_value` returns None when the property is not present in the - # data file, rather than raising. - value = ExampleUtils.get_value(aspect_data, name.lower()) - if value is None: - message.append(f"\t{name}: This property is not present in the data file.\n") - elif value.has_value(): + return needs_enterprise + + def outputValue(self, engine, aspect_data, name, message): + # A property the engine's data file does not have needs the + # Enterprise data file, and its value is not read. + if not ExampleUtils.has_property(engine, name): + message.append(f"\t{name}: {ExampleUtils.NEEDS_ENTERPRISE}\n") + return True + value = aspect_data.get(name.lower()) + if value.has_value(): message.append(f"\t{name}: {ExampleUtils.format_value(value.value())}\n") else: message.append(f"\t{name}: {value.no_value_message()}\n") + return False def main(argv): # Use the supplied paths for the data files, or find the device detection # and IP intelligence files in the project space. device_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(HASH_DATAFILE_NAME) - ip_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(ASN_DATAFILE_NAME) + ip_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. logger = Logger(min_level="info") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index 4785249e..a9e92fb9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -28,7 +28,9 @@ # # We create a pipeline using the on-premise engine, deserialize the # YAML evidence records, pass each entry to the pipeline and write the -# resolved properties out to another YAML file. +# resolved properties out to another YAML file. By default the free Lite +# data file is used, which is downloaded when it is missing, and Country, +# which only the Enterprise data file has, is recorded as needing it. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -42,12 +44,17 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME # Number of evidence records to process in this example. RECORD_LIMIT = 100 +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-offlineprocessing.py&utm_term=enterprise-data-file" + + class OfflineProcessing(): def run(self, data_file, evidence_yaml, logger, output): @@ -71,6 +78,7 @@ def run(self, data_file, evidence_yaml, logger, output): ExampleUtils.check_data_file(pipeline, logger) records = 0 + needs_enterprise = False with open(evidence_yaml, "r") as fin: for evidence in yaml.load_all(fin): if records >= RECORD_LIMIT: @@ -78,9 +86,14 @@ def run(self, data_file, evidence_yaml, logger, output): # write the yaml document separator output("---") - self.analyse_evidence(evidence, pipeline, output, yaml) + needs_enterprise |= self.analyse_evidence( + evidence, pipeline, output, yaml) records += 1 + if needs_enterprise: + # A comment keeps the output a valid YAML stream. + output("# " + ExampleUtils.get_enterprise_note(CONTACT_US)) + # write the yaml document end marker output("...") logger.log("info", f"Processed {records} evidence records") @@ -106,16 +119,20 @@ def analyse_evidence(self, evidence, pipeline, output, yaml): # Add the values that we want to store against the record. result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") + # Only the Enterprise data file has Country. + result["ip.country"] = ExampleUtils.get_enterprise_value( + pipeline.flow_elements_list["ip"], ip_data, "country") buffer = io.BytesIO() yaml.dump(result, buffer) output(buffer.getvalue().decode("utf-8").rstrip()) + return result["ip.country"] == ExampleUtils.NEEDS_ENTERPRISE def main(argv): - # Use the supplied path for the data file or find the enterprise file - # in the project space. - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + # Use the supplied path for the data file or find the free Lite file in + # the project space, which is downloaded when it is missing. + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) evidence_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(EVIDENCE_FILE_NAME) # Configure a logger to output to the console. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 1d910d81..e1001011 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -35,9 +35,12 @@ # # The native engine holds the Python GIL for the duration of each lookup, so # threads cannot run lookups in parallel. Parallelism therefore uses worker -# processes, each building its own pipeline. With the LowMemory profile the -# memory mapped data file is shared between the processes by the operating -# system, so a large enterprise data file is not duplicated per worker. +# processes, each building its own pipeline. With the default MaxPerformance +# profile each worker loads the whole data file into its own memory, so the +# memory used grows with the number of workers. With -p LowMemory each worker +# reads the data file from disk as it needs it, and the operating system's +# file cache holds one copy for all the workers, so a large Enterprise data +# file is not held once per worker. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -55,13 +58,17 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME # The IP intelligence properties used to measure performance. Restricting the # work to a single component keeps the benchmark focused on lookup speed. PERFORMANCE_PROPERTY = "registeredcountry" +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-performance_console.py&utm_term=enterprise-data-file" + # The number of records processed before timing starts, so that any lazy # initialisation is not included in the timed run. WARM_UP_RECORDS = 100 @@ -214,7 +221,9 @@ def main(argv): help="Performance profile: MaxPerformance (default) or LowMemory") args = ap.parse_args(argv) - data_file = args.data_file or ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + # By default the free ASN data file is used, which is far smaller than the + # Lite file and is downloaded when it is missing. + data_file = args.data_file or ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = args.evidence_file or ExampleUtils.find_file(EVIDENCE_FILE_NAME) logger = Logger(min_level="info") @@ -227,6 +236,9 @@ def main(argv): if args.json_output: with open(args.json_output, "w") as fout: json.dump(results, fout) + if ExampleUtils.is_free_data_file_at(data_file): + print("") + print(ExampleUtils.get_enterprise_offer(CONTACT_US)) else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index 0ada1f94..df742cdb 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -29,6 +29,8 @@ from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME +from .live_cloud import find_resource_key, no_resource_key + # The tests run against the freely available ASN data file the data # repository links to, so they only rely on properties that the free data # file provides (location properties such as Latitude, Longitude and Areas @@ -36,7 +38,14 @@ data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = ExampleUtils.find_file(EVIDENCE_FILE_NAME) truth_file = ExampleUtils.find_file("geoip_comparison_evidence.csv") -resource_key = ExampleUtils.get_resource_key() +resource_key = find_resource_key() + +NO_IP_KEY = no_resource_key( + "the ip product", + "IP Intelligence properties such as RegisteredName, and their metadata") +NO_MIXED_KEY = no_resource_key( + "the device and ip products", + "device detection and IP Intelligence properties together") # The mixed examples combine IP intelligence with device detection, which is # an optional dependency. Their tests only run when it is installed. @@ -146,23 +155,23 @@ def test_onpremise_compare_console(self): # header + up to 20 records self.assertGreater(len(output.getvalue().splitlines()), 1) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_gettingstarted_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.gettingstarted_console import GettingStartedConsole example = GettingStartedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_metadata_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.metadata_console import MetadataConsole example = MetadataConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_getallproperties_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.getallproperties_console import GetAllPropertiesConsole example = GetAllPropertiesConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) @@ -176,9 +185,10 @@ def test_onpremise_mixed_console(self): example.run(hash_file, data_file, Logger(min_level="error"), lambda x: None) @unittest.skipUnless( - device_detection_available and bool(resource_key), - "device detection package or resource_key not available") + device_detection_available, "device detection package not available") def test_cloud_mixed_console(self): + if not resource_key: + self.fail(NO_MIXED_KEY) from fiftyone_ipintelligence_examples.cloud.mixed_console import MixedConsole example = MixedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py new file mode 100644 index 00000000..e417bd79 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -0,0 +1,133 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Runs the on-premise examples as they run from a fresh clone, with no data +# file setting and no licence key, so that they use the free data files. +# Each must finish, show the properties only the Enterprise data file has as +# needing it, and give the link to ask for a copy. + +import io +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils +from fiftyone_ipintelligence_examples.onpremise import compare_console +from fiftyone_ipintelligence_examples.onpremise import countries_console +from fiftyone_ipintelligence_examples.onpremise import datafileupdate_console +from fiftyone_ipintelligence_examples.onpremise import gettingstarted_console +from fiftyone_ipintelligence_examples.onpremise import metadata_console +from fiftyone_ipintelligence_examples.onpremise import mixed_console +from fiftyone_ipintelligence_examples.onpremise import offlineprocessing +from fiftyone_ipintelligence_examples.onpremise import performance_console + +# The settings the examples read, which a fresh clone does not have. +SETTINGS = ["license_key", "LICENSE_KEY", ExampleUtils.DATA_FILE_ENV_VAR] + +# Options that keep the performance example short. They leave the data file +# to the example. +PERFORMANCE_OPTIONS = ["--record_limit", "200", "--workers", "1"] + +# The mixed example combines IP intelligence with device detection, which +# is an optional dependency. +try: + import fiftyone_devicedetection_onpremise # noqa: F401 + device_detection_available = True +except ImportError: + device_detection_available = False + + +def run_without_settings(main, argv=None): + """Runs an example's main with the settings taken out of the + environment, and returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + main(argv or []) + return printed.getvalue() + + +class ExamplesWithoutSettingsTests(unittest.TestCase): + + def test_gettingstarted_console(self): + printed = run_without_settings(gettingstarted_console.main) + self.assertIn("Country: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), + printed) + + def test_countries_console(self): + printed = run_without_settings(countries_console.main) + self.assertIn( + "Listing the countries an IP address is most likely in needs the " + "CountryCodesGeographical and CountryCodesPopulation properties", + printed) + self.assertIn(countries_console.CONTACT_US, printed) + + def test_metadata_console(self): + printed = run_without_settings(metadata_console.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), + printed) + + def test_offlineprocessing(self): + printed = run_without_settings(offlineprocessing.main) + self.assertIn("ip.country: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + "# " + ExampleUtils.get_enterprise_note( + offlineprocessing.CONTACT_US), + printed) + + def test_compare_console(self): + printed = run_without_settings(compare_console.main) + self.assertIn( + "Comparing known locations with IP Intelligence needs the " + "Latitude, Longitude, LocationConfidence and Country properties", + printed) + self.assertIn(compare_console.CONTACT_US, printed) + + def test_performance_console(self): + printed = run_without_settings( + performance_console.main, PERFORMANCE_OPTIONS) + self.assertIn( + ExampleUtils.get_enterprise_offer(performance_console.CONTACT_US), + printed) + + def test_datafileupdate_console(self): + printed = run_without_settings(datafileupdate_console.main) + self.assertIn("The data file in use is the", printed) + self.assertIn(datafileupdate_console.CONTACT_US, printed) + + @unittest.skipUnless( + device_detection_available, "device detection is not installed") + def test_mixed_console(self): + printed = run_without_settings(mixed_console.main) + self.assertIn(ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(mixed_console.CONTACT_US), + printed) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index 3bc4fbd3..459f1a48 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -37,6 +37,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_examples {tty:--color=yes} {posargs} pass_env = + _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key IPINTELLIGENCEDATAFILE diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 37b45904..9f15784e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -57,7 +57,7 @@ def __init__( polling_interval=30, update_time_maximum_randomisation=10, create_temp_data_copy=True, - performance_profile="LowMemory", + performance_profile="Balanced", reuse_temp_file=False, concurrency=multiprocessing.cpu_count(), update_on_start=False, @@ -84,7 +84,9 @@ def __init__( @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py index adb27093..182d39a8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py @@ -62,7 +62,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py new file mode 100644 index 00000000..65211ba7 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py @@ -0,0 +1,118 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule import ConfigIpiSwig +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import IpIntelligenceOnPremise +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise_pipelinebuilder import IpIntelligenceOnPremisePipelineBuilder + +from .constants import TestConstants + +# The free ASN data file, which needs no licence key. +DATA_FILE = os.path.normpath(TestConstants.get_asn_data_file_path()) + +# The collections whose settings the configuration can be asked for. The +# getters for components, properties, values and profiles hand Python a +# setting the configuration owns, which Python then frees, corrupting the +# heap, so those four are left out. +COLLECTIONS = ( + "Strings", "Maps", "Graphs", "ProfileGroups", "ProfileOffsets", + "PropertyTypes", "Graph") + +# Each profile name the engine accepts and the native setter it maps to. +PROFILES = { + "LowMemory": "setLowMemory", + "MaxPerformance": "setMaxPerformance", + "Balanced": "setBalanced", + "BalancedTemp": "setBalancedTemp", + "HighPerformance": "setHighPerformance", +} + + +def collection_settings(config): + """ + Whether each collection is loaded into memory and how many items it + caches, which are what a performance profile sets. + """ + settings = {} + for name in COLLECTIONS: + collection = getattr(config, "get" + name)() + settings[name] = (collection.getLoaded(), collection.getCapacity()) + return settings + + +def profile_settings(setter): + """The collection settings the native code gives a profile.""" + config = ConfigIpiSwig() + getattr(config, setter)() + return collection_settings(config) + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free ASN data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class PerformanceProfileDefaultTests(unittest.TestCase): + """ + The engine keeps no name for the profile in use, so these tests compare + the collection settings of the configuration the engine gives the + native code with the settings each profile has there. MaxPerformance + and LowMemory have the same collection settings, and Balanced and + BalancedTemp do too, so neither pair can be told apart this way. + """ + + def _engine_settings(self, **settings): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", **settings) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + self.assertIsNotNone(engine.engine) + return collection_settings(engine.config) + + def test_balanced_and_low_memory_differ(self): + self.assertNotEqual( + profile_settings("setBalanced"), profile_settings("setLowMemory")) + + def test_engine_defaults_use_balanced(self): + self.assertEqual( + profile_settings("setBalanced"), self._engine_settings()) + + def test_builder_defaults_use_balanced(self): + pipeline = IpIntelligenceOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, + licence_keys="").build() + self.addCleanup(pipeline.close) + engine = pipeline.get_element("ip") + self.assertEqual( + profile_settings("setBalanced"), + collection_settings(engine.config)) + + def test_each_named_profile_is_used(self): + for name, setter in PROFILES.items(): + with self.subTest(name): + self.assertEqual( + profile_settings(setter), + self._engine_settings(performance_profile=name)) diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py index 7f89e1d0..a4c4934c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py @@ -45,7 +45,18 @@ def read(file_name): "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=documentation", "Source": "https://github.com/51Degrees/51degrees-python", }, - description=("Shared code used by the cloud and on-premise variants of the 51Degrees IP Intelligence engines for the 51Degrees Pipeline API."), + description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. This package holds the code shared by the cloud and on-premise engines for the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo."), + keywords=[ + "51degrees", + "ip-intelligence", + "ip-geolocation", + "geoip", + "ip-lookup", + "asn", + "ip-to-country", + "maxmind", + "ipinfo" + ], long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.10", diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini index 2bf335a2..98972fd0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini @@ -12,6 +12,12 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_shared {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini index 11283b9e..b055ee03 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini @@ -12,6 +12,10 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_translation/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_translation {tty:--color=yes} {posargs} pass_env = diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 1af0c1c0..bbbdb0bb 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -6,8 +6,9 @@ param ( ) if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run." - return + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys diff --git a/location/tox.ini b/location/tox.ini index 3cd28ff4..504de885 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -8,6 +8,11 @@ wheel_build_env = .pkg deps = pytest>=6 pytest-cov + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../pipeline/fiftyone_pipeline_core/ + -e../pipeline/fiftyone_pipeline_engines/ + -e../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_location {tty:--color=yes} {posargs} pass_env = diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index 0ee466aa..5539b7ea 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,6 +5,12 @@ param ( [Hashtable]$Keys ) +if (!$Keys.TestResourceKey) { + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." +} + # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -16,9 +22,23 @@ param ( # not a licence key for the cloud service, so the examples' licence-key tests # failed with it. They are not given it, and skip themselves as they did when # pipeline was its own repository, until a cloud licence key is passed. -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" +# +# The 51Did package runs only its tests marked live here, as its others ran +# in the unit step. Its tier tests need a key for another tier, which only +# the Cloud 51Did workflow has. Only a package whose tox.ini passes +# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which +# build-project.ps1 copies, and the publish workflow's test job does not run +# that script, so it is copied here too. +& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "live and not tier"' +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/pipeline/ci/run-unit-tests.ps1 b/pipeline/ci/run-unit-tests.ps1 index c5af008e..441a2bcc 100644 --- a/pipeline/ci/run-unit-tests.ps1 +++ b/pipeline/ci/run-unit-tests.ps1 @@ -3,7 +3,16 @@ param ( [string]$RepoName ) -$packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" -./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +# The tests marked live call the cloud service, and no key reaches the unit +# step, so they are left to run-integration-tests.ps1. Only a package whose +# tox.ini passes PYTEST_ADDOPTS sees the selection. +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "not live"' +try { + $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" + ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index cd8b5b28..58b3ec35 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -90,8 +90,10 @@ then open . Set `PORT` to listen on another port. .venv/bin/python -m pytest ``` -The tests that call the cloud service skip themselves when no resource key -is set. +The tests that call the cloud service with a resource key fail when none is +set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY`, +`_51DEGREES_RESOURCE_KEY_BESPOKE` and `resource_key`, in that order. The +tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py index 74d791c1..60ec92d8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py @@ -29,18 +29,21 @@ from fiftyone_pipeline_examples.example_utils import ExampleUtils +from live_cloud import find_resource_key, no_resource_key + +NO_KEY = no_resource_key( + "the device, ip and countrynamestranslated products", + "device detection and IP Intelligence properties, and the translated " + "country names", several=True) + # The ways a test can reach the cloud service, as the keyword arguments the -# examples' create_pipeline takes. Each is skipped when its key is not set. +# examples' create_pipeline takes, each passed through credentials(). A test +# with a resource key fails when none is set. A test with a licence key is +# skipped when none is set, because CI has no cloud licence key to give it. KEYS = [ pytest.param( - {"resource_key": ExampleUtils.get_resource_key()}, - id="resource key", - marks=pytest.mark.skipif( - not ExampleUtils.get_resource_key(), - reason=f"Set {ExampleUtils.RESOURCE_KEY_ENV_VAR} to a resource " - "key carrying device detection and IP intelligence " - "properties to run the cloud tests with a resource " - "key.")), + {"resource_key": find_resource_key()}, + id="resource key"), pytest.param( {"license_key": ExampleUtils.get_license_key()}, id="licence key", @@ -53,6 +56,15 @@ ] +def credentials(keys): + """The keyword arguments from KEYS, failing the test when they hold a + resource key that is not set.""" + + if "resource_key" in keys and not keys["resource_key"]: + pytest.fail(NO_KEY, pytrace=False) + return keys + + def cell_after(page, label): """ The text a browser shows in the table cell after the one holding label, diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py index 190d940e..f4cb7bc4 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py @@ -34,7 +34,7 @@ GettingStartedConsole, create_pipeline) from fiftyone_pipeline_examples.example_utils import ExampleUtils -from helpers import KEYS +from helpers import KEYS, credentials SEPARATOR = "=" * 79 @@ -45,7 +45,7 @@ def output(request): pipeline = create_pipeline( Logger(min_level="info"), cloud_endpoint=ExampleUtils.get_cloud_endpoint(), - **request.param) + **credentials(request.param)) GettingStartedConsole().run(pipeline, lines.append) return "\n".join(lines) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py index c3f7eb6d..fa66520c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py @@ -37,7 +37,7 @@ create_app, create_pipeline) from fiftyone_pipeline_examples.example_utils import ExampleUtils -from helpers import KEYS, cell_after, country_options +from helpers import KEYS, cell_after, country_options, credentials DESKTOP_CHROME = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " @@ -51,7 +51,7 @@ @pytest.fixture(scope="module", params=KEYS) def key(request): - return request.param + return credentials(request.param) @pytest.fixture(scope="module") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index 1fae6a61..56e9bf52 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -29,15 +29,16 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_examples {tty:--color=yes} {posargs} # tox runs the tests in a clean environment and strips anything it is not -# told to keep, so without this the cloud tests never see a key and skip -# themselves on every run, which reads as a pass. resource_key and -# license_key are the names common-ci sets, from the secrets named in -# ci/run-integration-tests.ps1. The _51DEGREES names are the ones the -# examples document and the Examples workflow sets. The last three name a -# cloud service other than the public one: the first two are read by the -# examples, the third by the cloud request engine underneath them. +# told to keep, so without this the cloud tests never see a key and fail on +# every run. resource_key and license_key are the names common-ci sets, from +# the secrets named in ci/run-integration-tests.ps1. The _51DEGREES names are +# the ones the examples document, the Examples workflow sets and +# tests/live_cloud.py reads. The last three name a cloud service other than the +# public one: the first two are read by the examples, the third by the cloud +# request engine underneath them. pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY resource_key license_key diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py index afafa240..15854691 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -24,9 +24,10 @@ The callback URL the client script posts to for updated JSON. The protocol and host usually come from the request, and the script holds the URL in a single-quoted string, so the URL is checked before it is written into the -script. A URL that is not valid is refused. In the user information, in -anything after the host and port, and in the endpoint, the characters a URL -cannot hold or that could end the string or the script are percent-encoded. +script. A URL that is not valid, including one whose protocol is not http or +https, is refused. In anything after the authority, and in the endpoint, the +characters a URL cannot hold or that could end the string or the script are +percent-encoded. See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url """ @@ -58,23 +59,26 @@ r"[\t-\r \u0085\u00a0\u1680\u2000-\u200a" r"\u2028\u2029\u202f\u205f\u3000]*") -# A scheme as RFC 3986 section 3.1 defines one, being a letter followed by -# letters, digits, '+', '-' and '.'. -SCHEME_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") +# The schemes the script can send its request with. It uses the URL only with +# fetch and XMLHttpRequest, which reach a server with no other scheme. +SCHEMES = ("http", "https") -# Where the authority ends, which is at the first '/', '?' or '#' of the -# host. -AUTHORITY_END_PATTERN = re.compile(r"[/?#]") +# Lower-cases the ASCII letters A to Z and nothing else. +_ASCII_LOWER = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + +# The characters the host value must not hold. +FORBIDDEN_PATTERN = re.compile(r"[?#@]") # A label of a host name, being ASCII letters, digits, '-' and '_'. LABEL_PATTERN = re.compile(r"[A-Za-z0-9_-]+") -# An IPv6 zone identifier, being unreserved characters as RFC 3986 section -# 2.3 defines them. -ZONE_PATTERN = re.compile(r"[A-Za-z0-9._~-]+") +# A last label that makes the host a number, being only digits, or 0x or 0X +# and any number of hexadecimal digits. +NUMBER_LABEL_PATTERN = re.compile(r"[0-9]+|0[xX][0-9A-Fa-f]*") -# A port as it follows the host, being ':' and any number of digits. -PORT_PATTERN = re.compile(r":[0-9]*") +# A port as it follows the host, being ':' and at least one digit. +PORT_PATTERN = re.compile(r":[0-9]+") # One group of an IPv6 address. HEX_GROUP_PATTERN = re.compile(r"[0-9A-Fa-f]{1,4}") @@ -105,24 +109,56 @@ def is_blank(text): or BLANK_PATTERN.fullmatch(text) is not None) +def ascii_lower(text): + + """! + The text with the ASCII letters A to Z in lower case and every other + character as it is, so that no letter outside ASCII can stand in for an + ASCII one, as str.casefold would let the long s stand in for "s". + + @type text: str + @param text: The text + @rtype: str + @return: The text with A to Z in lower case + """ + + return text.translate(_ASCII_LOWER) + + +def is_scheme(protocol): + + """! + Whether the protocol is http or https, ignoring the case of its ASCII + letters only. + + @type protocol: str + @param protocol: The protocol + @rtype: bool + @return: True if the protocol is http or https + """ + + return ascii_lower(protocol) in SCHEMES + + def callback_url(protocol, host, endpoint): """! Build the callback URL from its parts and check it. One slash separates the host and the endpoint, being added when neither has one and removed when both do. The protocol and the host are written in lower case and - the rest as given, apart from the percent-encoding. + the rest as given, apart from the percent-encoding. The host value holds + no '?', '#' or '@'. @type protocol: str - @param protocol: The protocol, such as https + @param protocol: The protocol, http or https in any case @type host: str - @param host: The host, which can start with user information and end - with a port, a path, a query or a fragment + @param host: The host, which can end with a port and a path @type endpoint: str @param endpoint: The endpoint @rtype: CallbackUrl @return: The callback URL, or None when the protocol, host or endpoint - is not set + is not set. The URL of an invalid one is as it was built, for the log, + with '***' in place of everything in the host before its last '@'. """ if is_blank(protocol) or is_blank(host) or is_blank(endpoint): @@ -133,54 +169,37 @@ def callback_url(protocol, host, endpoint): endpoint = "/" + endpoint elif host_has_slash and endpoint_has_slash: endpoint = endpoint[1:] - built = protocol + "://" + host + endpoint - if SCHEME_PATTERN.fullmatch(protocol) is None: + at = host.rfind("@") + built = (protocol + "://" + ("***" + host[at:] if at != -1 else host) + + endpoint) + if not is_scheme(protocol) or FORBIDDEN_PATTERN.search(host): return CallbackUrl(False, built) - match = AUTHORITY_END_PATTERN.search(host) - end = len(host) if match is None else match.start() - authority = _checked_authority(host[:end]) + end = host.find("/") + end = len(host) if end == -1 else end + authority = _checked_host_port(host[:end]) if authority is None: return CallbackUrl(False, built) return CallbackUrl( True, - protocol.lower() + "://" + authority + ascii_lower(protocol) + "://" + authority + _encoded(host[end:]) + _encoded(endpoint)) -def _checked_authority(authority): - - """! - The user information, host and port as they are written into the URL, - or None when they are not valid. The user information ends at the first - '@', so a second '@' is part of the host and refused. - """ - - user, at, host_port = authority.partition("@") - if not at: - return _checked_host_port(authority) - host_port = _checked_host_port(host_port) - if host_port is None: - return None - return _encoded(user) + "@" + host_port - - def _checked_host_port(host_port): """! The host and optional port in lower case, or None when they are not - valid. The host is an IPv6 address in brackets with an optional zone - identifier, or a name of labels separated by dots with a dot allowed at - the end. The port is ':' followed by digits up to 65535, or by nothing. + valid. The host is an IPv6 address in brackets, or a name of labels + separated by dots with a dot allowed at the end, which is an IPv4 address + when its last label is a number. The port is ':' followed by digits up to + 65535. """ if host_port.startswith("["): close = host_port.find("]") if close == -1: return None - address, separator, zone = host_port[1:close].partition("%25") - if separator and ZONE_PATTERN.fullmatch(zone) is None: - return None - if not _is_ipv6(address): + if not _is_ipv6(host_port[1:close]): return None host = host_port[:close + 1] port = host_port[close + 1:] @@ -188,9 +207,13 @@ def _checked_host_port(host_port): host, colon, port = host_port.partition(":") port = colon + port name = host[:-1] if host.endswith(".") else host - for label in name.split("."): + labels = name.split(".") + for label in labels: if LABEL_PATTERN.fullmatch(label) is None: return None + if (NUMBER_LABEL_PATTERN.fullmatch(labels[-1]) + and IPV4_PATTERN.fullmatch(host) is None): + return None if port and not _is_port(port): return None return host.lower() + port @@ -200,7 +223,7 @@ def _is_port(text): """! Whether the text is ':' followed by digits whose value is no more than - 65535, or by nothing. + 65535. """ if PORT_PATTERN.fullmatch(text) is None: diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 627ca7d8..f7067647 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -98,7 +98,8 @@ def filter(self) -> None: if self.evidence_filtered: return - for key in self.evidence: + # Loop over a copy of the keys, as keys are removed on the way. + for key in list(self.evidence): if not any([ flow_element.filter_evidence_key(key) for flow_element diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 31d063a7..bdb85c50 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -38,7 +38,7 @@ from .evidence_keyfilter import EvidenceKeyFilter from .elementdata_dictionary import ElementDataDictionary from .constants import Constants -from .callback_url import callback_url +from .callback_url import callback_url, is_scheme # Evidence keys the rendered script is not configured with. Both are @@ -255,9 +255,10 @@ def __init__(self, settings = {} ): * starting with a digit) and not a reserved word. An invalid * configured name raises a ValueError, and an invalid requested name * is ignored with a warning logged. - * @param {string} options.protocol The protocol ("http" or "https") - * used by the client side callback url. - * This can be overriden with header.protocol evidence + * @param {string} options.protocol The protocol of the client side + * callback url, http or https in any case of its ASCII letters. Any + * other configured protocol raises a ValueError. Left out, the + * header.protocol evidence is used, or https when there is none. * @param {string} options.host The host of the client side * callback url. This can be overriden with header.host evidence. * @param {string} options.endpoint The endpoint of the client side @@ -286,6 +287,15 @@ def __init__(self, settings = {} ): "JavaScript identifier (ASCII letters, digits, '_' and '$', " "not starting with a digit) and not a reserved word.") self.settings['_protocol'] = settings["protocol"] if "protocol" in settings else None + + # A protocol given here is used for every request, so one the script + # cannot send its request with is refused now rather than logged on + # the first request. + protocol = self.settings['_protocol'] + if protocol and not (isinstance(protocol, str) and is_scheme(protocol)): + raise ValueError( + "The JavaScript builder's protocol setting must be http or " + "https.") self.settings['_host'] = settings["host"] if "host" in settings else None self.settings['_endpoint'] = settings["endpoint"] if "endpoint" in settings else '' self.settings['_enableCookies'] = settings["enable_cookies"] if "enable_cookies" in settings else True diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py index 9048401d..29e8be4d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -69,21 +69,22 @@ def test_missing_when_a_part_is_empty(self): @parameterized.expand([ ("EXAMPLE.com", "https://example.com/51dpipeline/json"), ("example.com:8080", "https://example.com:8080/51dpipeline/json"), - ("example.com:", "https://example.com:/51dpipeline/json"), ("example.com:65535", "https://example.com:65535/51dpipeline/json"), ("a_b.com", "https://a_b.com/51dpipeline/json"), ("-ab.com", "https://-ab.com/51dpipeline/json"), ("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + ("1.2.3.4", "https://1.2.3.4/51dpipeline/json"), + ("1.2.3.4:8080", "https://1.2.3.4:8080/51dpipeline/json"), + ("1.2.3.4/path", "https://1.2.3.4/path/51dpipeline/json"), + ("0.0.0.0", "https://0.0.0.0/51dpipeline/json"), + ("101.2.3.4", "https://101.2.3.4/51dpipeline/json"), + ("example.co1", "https://example.co1/51dpipeline/json"), + ("123.example.com", "https://123.example.com/51dpipeline/json"), + ("0x7f.example.com", "https://0x7f.example.com/51dpipeline/json"), ("[::1]", "https://[::1]/51dpipeline/json"), ("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), - ("[fe80::1%25eth0]", "https://[fe80::1%25eth0]/51dpipeline/json"), - ("user:pw@example.com", - "https://user:pw@example.com/51dpipeline/json"), - ("u x@example.com", "https://u%20x@example.com/51dpipeline/json"), - ("u\tx@example.com", "https://u%09x@example.com/51dpipeline/json"), ("example.com.", "https://example.com./51dpipeline/json"), ("example.com/path", "https://example.com/path/51dpipeline/json"), - ("example.com?x", "https://example.com?x/51dpipeline/json"), ("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), ("example.com/a%zzb", "https://example.com/a%25zzb/51dpipeline/json"), @@ -110,7 +111,6 @@ def test_accepts_the_host(self, host, expected): ("[::1",), ("[v1.x]",), ("[1.2.3.4]",), - ("a@b@example.com",), ("a..b.com",), ("%41.com",), ("a\\b.com",), @@ -137,26 +137,202 @@ def test_accepts_the_host(self, host, expected): def test_refuses_the_host(self, host): self.assertFalse(_url("https", host).valid, repr(host)) + @parameterized.expand([ + ("[fe80::1%25]",), + ("[fe80::1%25'+alert(1)+']",), + ("[fe80::1%eth0]",), + ("[fe80::1%25eth%0]",), + ("[::1]x",), + ("[::1]80",), + ("[::1]:80x",), + ("[::1]]",), + ]) + def test_refuses_the_zone_or_the_text_after_the_bracket(self, host): + """Inside the brackets only an IPv6 address, optionally followed by + '%25' and a zone of unreserved characters, and after them only a + port. The zone check is what keeps a quote out of the script for a + bracketed host.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + def test_checks_the_port_by_its_value(self): + """A port goes by its value, so leading zeros are allowed, and a + port too long to read is refused rather than read as small enough.""" + + self.assertEqual( + _valid("https://example.com:00080/51dpipeline/json"), + _url("https", "example.com:00080")) + self.assertEqual( + _valid("https://example.com:0065535/51dpipeline/json"), + _url("https", "example.com:0065535")) + for host in [ + "example.com:0065536", + "example.com:123456", + "example.com:4294967296", + "example.com:99999999999999999999"]: + self.assertFalse(_url("https", host).valid, host) + + def test_encodes_the_characters_rule_8_lists(self): + """Each character rule 8 lists, after the host and in the endpoint, + and a '%' without two hexadecimal digits after it, even at the + end.""" + + self.assertEqual( + _valid("https://example.com/%22%5E%60%7B%7C%7D" + "/j%22%5E%60%7B%7C%7D"), + callback_url("https", 'example.com/"^`{|}', '/j"^`{|}')) + self.assertEqual( + _valid("https://example.com/a%25/51dpipeline/json"), + _url("https", "example.com/a%")) + self.assertEqual( + _valid("https://example.com/a%254/51dpipeline/json"), + _url("https", "example.com/a%4")) + self.assertEqual( + _valid("https://example.com/a%41/51dpipeline/json"), + _url("https", "example.com/a%41")) + self.assertEqual( + _valid("https://example.com/json%25"), + callback_url("https", "example.com", "/json%")) + + @parameterized.expand([ + ("[fe80::1%25eth0]",), + ("[fe80::1%25eth0]:8080",), + ("[FE80::1%25ETH0]",), + ("[fe80::1%251]",), + ("[::1%25lo]",), + ("[fe80::1%]",), + ("[fe80::1%25eth0]/path",), + ]) + def test_refuses_the_zone_identifier(self, host): + """Rule 4 refuses an IPv6 zone identifier.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("user@example.com",), + ("user:pw@example.com",), + ("@example.com",), + (":@example.com",), + ("user@example.com:8080",), + ("user@[::1]",), + ("user@example.com/path",), + ("u%41@example.com",), + ("u x@example.com",), + ("u\tx@example.com",), + ("u\"x@example.com",), + ("u'x@example.com",), + ("u\\x@example.com",), + ("u\nx@example.com",), + ("u\u0001x@example.com",), + ("a@b@example.com",), + ("example.com/a@b",), + ("exa mple.com/a@b",), + ("user:12/ss@example.com",), + ("user:pa/ss@example.com",), + ("user:pa?ss@example.com",), + ]) + def test_refuses_the_user_information(self, host): + """Rule 3 refuses an '@' anywhere in the host value.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.com?x",), + ("example.com#x",), + ("example.com?",), + ("example.com#",), + ("example.com:8080?x",), + ("[::1]#x",), + ("example.com/path?x",), + ("example.com/path#x",), + ("example.com?');alert(1);//",), + ("example.com#');alert(1);//",), + ]) + def test_refuses_the_query_or_fragment(self, host): + """Rule 2 refuses a '?' or '#' anywhere in the host value.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.123",), + ("example.0x1f",), + ("example.0X",), + ("1.2.3.999",), + ("1.2.3",), + ("127.1",), + ("0x7f.0.0.1",), + ("01.2.3.4",), + ("00.2.3.4",), + ("example.123.",), + ("1.2.3.4.",), + ("1.2.3.4.5",), + ("1",), + ("300.1.1.1",), + ("0x7f.1",), + ("1.2.3.4.:8080",), + ]) + def test_refuses_a_name_ending_in_a_number(self, host): + """Rule 4 reads a name whose last label is a number as an IPv4 + address written as four decimal numbers, each starting with 0 only + when it is 0 alone.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.com:",), + ("example.com:/path",), + ("[::1]:",), + ("https://example.com",), + ("http://example.com",), + ]) + def test_refuses_the_empty_port(self, host): + """Rule 5 refuses a port that is present but empty, so a protocol + written into the host gives the host "https" with an empty port.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("user:secret@example.com", + "https://***@example.com/51dpipeline/json"), + ("a@b@example.com", "https://***@example.com/51dpipeline/json"), + ("u@[::1]:80x", "https://***@[::1]:80x/51dpipeline/json"), + ("user:pa/ss@example.com", + "https://***@example.com/51dpipeline/json"), + ("user:pa?ss@example.com", + "https://***@example.com/51dpipeline/json"), + ("exa mple.com/a@b", "https://***@b/51dpipeline/json"), + ]) + def test_the_logged_url_holds_no_user_information(self, host, logged): + """Rule 6 shows *** in place of everything in the host before its + last '@', wherever in the host value that '@' is.""" + + self.assertEqual(CallbackUrl(False, logged), _url("https", host)) + def test_the_invalid_url_is_the_one_built(self): self.assertEqual( CallbackUrl(False, "https://exa mple.com/51dpipeline/json"), _url("https", "exa mple.com")) def test_checks_the_scheme(self): - for protocol in ["https", "http", "ftp", "h+t-p.s"]: + """Only http and https, ignoring the case of ASCII letters only, so + a letter outside ASCII that a case-insensitive comparison can match + to an ASCII one, such as the long s, is refused.""" + + for protocol in ["https", "http", "HtTp"]: self.assertTrue(_url(protocol, "example.com").valid, protocol) self.assertEqual( _valid("https://example.com/51dpipeline/json"), _url("HTTPS", "example.com")) - for protocol in ["1http", "h t", "x');alert(1);//", "a://b"]: - self.assertFalse(_url(protocol, "example.com").valid, protocol) + for protocol in [ + "ftp", "h+t-p.s", "ws", "javascript", "httpx", "https ", + "1http", "h t", "x');alert(1);//", "a://b", "http\u017f", + "HTTP\u017f"]: + self.assertFalse( + _url(protocol, "example.com").valid, repr(protocol)) @parameterized.expand([ ("example.com/');alert(1);//",), - ("example.com?');alert(1);//",), - ("example.com#');alert(1);//",), - ("u'x@example.com",), - ("u\\x@example.com",), + ('example.com/a"b',), ("example.com/a\\b",), ("example.com/",), ]) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py index f26acde6..21407b33 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_core.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py @@ -58,6 +58,24 @@ def testEvidenceKeyFilter(self): nullEvidence = testPipeline.flowdata.evidence.get("header.other-evidence") self.assertTrue(nullEvidence == None) + # Test that evidence no element accepts is removed without an error. + # The JavaScript and set header elements accept every key, so they are + # left out. + def testEvidenceKeyFilterRemovesKeys(self): + + pipeline = (PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}))\ + .add(ExampleFlowElement1())\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "test") + flowdata.evidence.add("header.accept-encoding", "gzip") + flowdata.evidence.add("header.host", "example.com") + flowdata.process() + self.assertEqual( + flowdata.evidence.get_all(), {"header.user-agent": "test"}) + # Test Getter methods def testGet(self): diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py index 92f95380..8342461f 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py @@ -151,7 +151,9 @@ def test_valid_name_from_evidence_minified(self): @parameterized.expand([ ["statement", "a;b", True], - ["leading_digit", "9bad", True], + # The name has a letter that is not a hex digit, so the random + # session id the script also carries can never contain it. + ["leading_digit", "9zz", True], ["quote", 'x"y', True], ["empty", "", False], # A reserved word is also an ordinary word in the script's comments, diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index 4c9302f4..9ac0e771 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -24,6 +24,8 @@ import unittest from unittest import mock +from parameterized import parameterized + from fiftyone_pipeline_core import javascriptbuilder from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder @@ -200,6 +202,51 @@ def test_refused_url_is_left_out_and_logged_once(self): self.assertEqual(1, len(errors), errors) self.assertIn("'https://exa mple.com/json'", errors[0]) + @parameterized.expand([ + ("user:secret@example.com", "'https://***@example.com/json'"), + ("a@b@example.com", "'https://***@example.com/json'"), + ("u@[::1]:80x", "'https://***@[::1]:80x/json'"), + ("user:pa/ss@example.com", "'https://***@example.com/json'"), + ("user:pa?ss@example.com", "'https://***@example.com/json'"), + ("exa mple.com/a@b", "'https://***@b/json'"), + ]) + def test_refused_host_is_logged_without_its_user_information( + self, host, logged): + """The log never holds what comes before the last '@' of the host + value. Each host has an element of its own because a refusal is + logged once per element.""" + + rendered, errors = _render_with_hosts([host]) + + self.assertNotIn("_url", rendered[0]) + self.assertEqual(1, len(errors), errors) + self.assertIn(logged, errors[0]) + self.assertNotIn("secret", errors[0]) + self.assertNotIn("'pa", errors[0]) + self.assertNotIn("pa/", errors[0]) + + def test_refused_configured_host_is_logged_without_user_information( + self): + """A host from the settings is refused and logged the same way.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json", + "host": "user:secret@example.com"}}).add_logger( + logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + pipeline.create_flowdata().process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + + self.assertNotIn("_url", captured.variables[0]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'https://***@example.com/json'", errors[0]) + self.assertNotIn("secret", errors[0]) + def test_refused_url_is_logged_on_one_line(self): """The URL is quoted and escaped in the log, so a line break in it cannot start a new log line.""" @@ -211,6 +258,33 @@ def test_refused_url_is_logged_on_one_line(self): self.assertNotIn(c, errors[0]) self.assertIn("a\\nb.com\\r\\u2028", errors[0]) + def test_refused_protocol_from_the_request_is_left_out_and_logged_once( + self): + """A protocol from the request that is neither http nor https gives + a script with no callback, and is logged once like any other URL + that is not valid.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json"}}).add_logger(logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + for _ in range(2): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.host", "example.com") + flowdata.evidence.add("header.protocol", "ftp") + flowdata.process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + + for variables in captured.variables: + self.assertNotIn("_url", variables) + self.assertFalse(variables["_updateEnabled"]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'ftp://example.com/json'", errors[0]) + def test_blank_host_gives_no_callback_and_is_not_logged(self): """A host made only of whitespace is not set, so the script has no callback and nothing is logged.""" @@ -232,5 +306,25 @@ def test_host_from_the_request_is_written_as_checked(self): "https://example.com/%27);alert(1);//json", rendered[0]["_url"]) +class JavaScriptBuilderProtocolSettingTests(unittest.TestCase): + + @parameterized.expand([ + ("ftp",), ("ws",), ("javascript",), ("https ",), ("http\u017f",), + ]) + def test_invalid_configured_protocol_is_refused(self, protocol): + """A protocol in the settings is used for every request, so one the + script cannot send its request with is refused when the builder is + created.""" + + with self.assertRaises(ValueError): + javascriptbuilder.JavascriptBuilderElement({"protocol": protocol}) + + @parameterized.expand([("http",), ("https",), ("HTTPS",)]) + def test_valid_configured_protocol_is_accepted(self, protocol): + element = javascriptbuilder.JavascriptBuilderElement( + {"protocol": protocol}) + self.assertEqual(protocol, element.settings["_protocol"]) + + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index fe51f236..3c2f369c 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -112,11 +112,12 @@ url_safe = fod_id.as_base64_url() # URL-safe alphabet, no padding, for a link ``` `from_base64` accepts the standard alphabet the cloud issues and the -URL-safe alphabet a page puts in a link, with or without padding. On an -identifier carrying a creator context the License Id field holds an -encrypted value that only 51Degrees can turn back into a licence -identifier, so `license_id` is the field's raw value and identifies -nothing outside 51Degrees. +URL-safe alphabet a page puts in a link, with or without padding. White +space around the value is ignored. A value with white space inside it, or +with characters from both alphabets, is refused. On an identifier carrying +a creator context the License Id field holds an encrypted value that only +51Degrees can turn back into a licence identifier, so `license_id` is the +field's raw value and identifies nothing outside 51Degrees. ### The usage an identifier was created for @@ -267,9 +268,9 @@ same whichever language parsed the bytes. | Status | Meaning | | --- | --- | | `PARSED` | A structurally valid 51Did. The signature has not been checked | -| `MISSING_INPUT` | `None`, an empty string or an empty buffer | +| `MISSING_INPUT` | `None`, an empty string, a string of white space alone or an empty buffer | | `INVALID_INPUT_TYPE` | Not a string (base64 reader) or not a bytes-like object (byte reader) | -| `INVALID_BASE64` | The text is not base64 in either alphabet | +| `INVALID_BASE64` | The text is not base64 in one alphabet, so there are no bytes. White space inside the text, and characters from both alphabets, report this | | `UNSUPPORTED_VERSION` | The first byte names an envelope version this package does not know | | `UNEXPECTED_END` | The data stopped in the middle of an envelope field | | `INVALID_DOMAIN_ENCODING` | The creator domain is not terminated or is longer than the OWID maximum | diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 6297b286..1a28e810 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -65,7 +65,7 @@ from ._owid import Version from ._layout import GUID_LENGTH, HEADER_LENGTH, MATCH_KEY_LENGTH -from .fod_id import DATE_EPOCH, FodId, _date_minutes +from .fod_id import DATE_EPOCH, FodId, _date_minutes, _trim_white_space from .id_type import IdType #: The public cloud API base, used when neither the ``endpoint`` argument @@ -934,10 +934,10 @@ def _ensure_parses(value: str) -> None: def _ensure_encoded_size(value: str) -> None: """Refuses obviously malformed text before it is parsed, a key is - fetched or the cloud is called. Surrounding whitespace is stripped - before measuring, as the reader strips it before decoding, so the two - measure the same characters.""" - if len(value.strip()) > _MAXIMUM_ENCODED_LENGTH: + fetched or the cloud is called. White space around the value is + removed before measuring, as the reader removes it before decoding, + so the two measure the same characters.""" + if len(_trim_white_space(value)) > _MAXIMUM_ENCODED_LENGTH: raise ValueError( "The identifier is far longer than any 51Did the cloud " "issues.") diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index a71e388f..0953b187 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -22,6 +22,7 @@ from __future__ import annotations +import re from datetime import datetime, timezone from enum import Enum from typing import NamedTuple, Optional, Tuple @@ -82,7 +83,9 @@ class FodIdParseStatus(Enum): MISSING_INPUT = "MissingInput" #: The input was supplied in a form this surface cannot read. INVALID_INPUT_TYPE = "InvalidInputType" - #: The string is not valid base 64, so there are no bytes to read. + #: The string is not valid base 64 in one alphabet, so there are no + #: bytes to read. White space inside the string, and characters from + #: both alphabets, report this. INVALID_BASE64 = "InvalidBase64" #: The first byte names an envelope version this package does not know. UNSUPPORTED_VERSION = "UnsupportedVersion" @@ -303,11 +306,17 @@ def try_from_base64(cls, value) -> FodIdParseResult: The cloud issues a 51Did in the standard alphabet with padding, and a page puts one in a link in the URL-safe alphabet (``-`` and ``_``) without padding. Both are accepted, with or without padding, by - normalising to the standard form before the envelope is read. - - The value may be anything at all, as external data is. ``None`` and - the empty string report :attr:`FodIdParseStatus.MISSING_INPUT`, - anything other than a string reports + normalising to the standard form before the envelope is read. White + space around the value is ignored. A value with white space or any + other character outside the alphabet inside it, or with characters + from both alphabets, reports + :attr:`FodIdParseStatus.INVALID_BASE64`, as + :meth:`to_standard_base64` explains. + + The value may be anything at all, as external data is. ``None``, the + empty string and a string of white space alone report + :attr:`FodIdParseStatus.MISSING_INPUT`, anything other than a string + reports :attr:`FodIdParseStatus.INVALID_INPUT_TYPE`, and every other failure names its reason. The signature is not checked. """ @@ -345,15 +354,33 @@ def from_base64(cls, base64: str) -> "FodId": return cls._from_read_or_raise(_read_base64(base64), "base64") @staticmethod - def to_standard_base64(value: str) -> str: + def to_standard_base64(value: str) -> Optional[str]: """Restores a base64 string in either alphabet to the standard alphabet with padding, which is the form the OWID library decodes. - ``-`` becomes ``+`` and ``_`` becomes ``/``, then ``==`` is added - when the length modulo 4 is 2 and ``=`` when it is 3. A value already - in the standard padded form is returned unchanged. + White space before and after the text is removed first, being every + character with the Unicode White_Space property, so a value read + from a file or a header with a trailing line break reads as the + clean value does. Then ``-`` becomes ``+`` and ``_`` becomes ``/``, + and ``==`` is added when the length modulo 4 is 2 and ``=`` when it + is 3. A value already in the standard padded form is returned + unchanged apart from that trim. + + Answers ``None`` for a text the 51Did specification says to refuse, + being one with white space or any other character outside the + alphabet left inside it, or one holding a character from each + alphabet, such as both ``+`` and ``-``. The issuer writes neither, + so either is a sign of a damaged value, such as a ``+`` a form + decoder turned into a space or a value converted only in part. """ - cleaned = value.strip().replace("-", "+").replace("_", "/") + text = _trim_white_space(value) + if _BASE64_CHARACTERS.fullmatch(text) is None: + return None + # Checked before the mapping below, which would hide a mix. + if (_has_any(text, _STANDARD_ONLY) and + _has_any(text, _URL_SAFE_ONLY)): + return None + cleaned = text.replace("-", "+").replace("_", "/") remainder = len(cleaned) % 4 if remainder == 2: cleaned += "==" @@ -365,9 +392,12 @@ def to_standard_base64(value: str) -> str: def to_base64_url(value: str) -> str: """Converts a base64 string in either alphabet to the URL-safe alphabet without padding, the inverse of :meth:`to_standard_base64`, - so a 51Did can be placed in a URL without further encoding. + so a 51Did can be placed in a URL without further encoding. White + space around the value is removed as :meth:`to_standard_base64` + removes it. """ - return value.strip().replace("+", "-").replace("/", "_").rstrip("=") + return _trim_white_space(value).replace("+", "-").replace( + "/", "_").rstrip("=") @classmethod def from_byte_array(cls, buffer: bytes) -> "FodId": @@ -546,12 +576,49 @@ def signature_status(self, public_pem: str) -> SignatureStatus: return self._owid.signature_status(public_pem) +#: Every character with the Unicode White_Space property, which is what the +#: 51Did specification means by white space. Written out because +#: ``str.strip()`` and ``str.isspace()`` also take U+001C to U+001F, which +#: do not have that property and so are outside the alphabet. +_WHITE_SPACE = ( + "\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000") + +#: The characters a 51Did text may hold once the white space around it is +#: removed, being both alphabets and the padding. Which alphabet the text +#: uses is checked separately, and where the padding sits is left to the OWID +#: reader's decoder. +_BASE64_CHARACTERS = re.compile(r"[A-Za-z0-9+/_=-]*") + +#: The characters for the values 62 and 63 that only the standard alphabet +#: uses. +_STANDARD_ONLY = "+/" + +#: The characters for the values 62 and 63 that only the URL-safe alphabet +#: uses. +_URL_SAFE_ONLY = "-_" + + +def _trim_white_space(value: str) -> str: + """The value without the white space before and after it.""" + return value.strip(_WHITE_SPACE) + + +def _has_any(text: str, characters: str) -> bool: + """Whether the text holds any of the characters.""" + return any(character in text for character in characters) + + def _read_base64(value) -> ParseResult: - """The OWID read of a base64 string in either alphabet. Anything that is - not a string goes to the OWID reader as given, so the reason it reports - (nothing supplied, or a type it cannot read) is the one carried.""" + """The OWID read of a base64 string in either alphabet. A string the + 51Did rules refuse reports invalid base 64, as the OWID reader does for + any other text it cannot decode. Anything that is not a string goes to + the OWID reader as given, so the reason it reports (nothing supplied, or + a type it cannot read) is the one carried.""" if isinstance(value, str): value = FodId.to_standard_base64(value) + if value is None: + return ParseResult(False, None, ParseStatus.INVALID_BASE64) return Owid.parse(value) diff --git a/pipeline/fiftyone_pipeline_did/tests/conftest.py b/pipeline/fiftyone_pipeline_did/tests/conftest.py new file mode 100644 index 00000000..8dd5e4f9 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/conftest.py @@ -0,0 +1,38 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The markers that keep the live cloud tests out of the unit test run. +# pipeline/ci/run-unit-tests.ps1 deselects "live", and +# pipeline/ci/run-integration-tests.ps1 runs "live and not tier", because no +# key for another tier reaches it. The Cloud 51Did workflow runs the tier +# tests with the Basic key. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + config.addinivalue_line( + "markers", + "tier: asserts what a key on one tier is entitled to, and needs the " + "key for that tier") diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py index 23d8fe90..6a41e265 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py @@ -20,25 +20,27 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -"""Live tests against the cloud, run only when a resource key is set. - -The key is read from ``resource_key``, the variable the repository's other -live tests and its CI use, or from ``_51DEGREES_RESOURCE_KEY`` as the -example reads it. Without either the tests are skipped, not failed. An -optional licence key is read from ``license_key`` or -``_51DEGREES_LICENSE_KEY``, and the endpoint from ``FOD_CLOUD_API_URL`` as -everywhere else. Every test here costs uses against the resource key. +"""Live tests against the 51Degrees cloud service. + +The resource key is read from the names in ``live_cloud.py``, and every +test fails when none is set. The key has to carry the FODid product, and +the CloudV5FODiD product for the marketing usages, which +``test_did_tier_live.py`` shows a Basic key does not get. An optional +licence key is read from ``license_key`` or ``_51DEGREES_LICENSE_KEY``, and +the endpoint from ``FOD_CLOUD_API_URL`` as everywhere else. Every test here +costs uses against the resource key. """ import asyncio import json import os -import sys import unittest import urllib.error import urllib.parse import urllib.request +import pytest + from fiftyone_pipeline_did import ( ContextResult, DidClient, @@ -49,11 +51,22 @@ ) from fiftyone_pipeline_did.did_client import USER_AGENT -RESOURCE_KEY = os.environ.get("resource_key") \ - or os.environ.get("_51DEGREES_RESOURCE_KEY") +from .live_cloud import answered_without, find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + +RESOURCE_KEY = find_resource_key() LICENCE_KEY = os.environ.get("license_key") \ or os.environ.get("_51DEGREES_LICENSE_KEY") +FODID = "the FODid product" +MARKETING = "the CloudV5FODiD product" + +NO_KEY = no_resource_key( + FODID + ", and " + MARKETING + " for the marketing usages", + "FODiD.IdProbGlobal and FODiD.IdProbLic under each id.usage", + several=True) + def run(coroutine): """Runs one client call to completion on a fresh event loop, as the @@ -62,14 +75,27 @@ def run(coroutine): return asyncio.run(coroutine) -@unittest.skipUnless( - RESOURCE_KEY, - "set resource_key (or _51DEGREES_RESOURCE_KEY) to run the live tests") +def null_reason(fodid): + """The reason the cloud gave for answering without a 51Did.""" + return fodid.get("idprobglobalnullreason") \ + or fodid.get("idproblicnullreason") + + +def no_creator_context(endpoint, detail): + """What to fail with when the host does not offer the creator context, + which the 51Degrees cloud does.""" + return ("The cloud service at {0} {1}, so the creator context could not " + "be checked. Point FOD_CLOUD_API_URL at a service that offers " + "it, or leave it unset for the 51Degrees cloud.").format( + endpoint, detail) + + class DidClientLiveTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.client = DidClient(RESOURCE_KEY, LICENCE_KEY) + def setUp(self): + if not RESOURCE_KEY: + self.fail(NO_KEY) + self.client = DidClient(RESOURCE_KEY, LICENCE_KEY) def create(self): """Creates a 51Did through the cloud ``json`` endpoint, the route @@ -88,10 +114,10 @@ def create(self): body = json.loads(response.read().decode("utf-8")) fodid = body.get("fodid") or {} value = fodid.get("idproblic") or fodid.get("idprobglobal") - self.assertTrue( - value, "the resource key must include a 51Did property " - "(idprobglobal or idproblic); the json answer was: " - + json.dumps(body)[:300]) + if not value: + self.fail(answered_without( + "a 51Did for id.usage=non-marketing", null_reason(fodid), + FODID)) return FodId.from_base64(value) def test_created_identifier_verifies_offline_and_through_the_cloud(self): @@ -108,8 +134,10 @@ def test_garbage_result_redeems_as_unreadable(self): try: result = run(self.client.redeem(fod_id, "not-base64url!!", "x")) except DidNotSupportedError as error: - self.skipTest("the host does not offer the creator context: " - + str(error)) + self.fail(no_creator_context( + self.client.endpoint, + "answered that it does not offer the creator context (" + + str(error) + ")")) self.assertEqual(200, result.status_code) self.assertEqual(ContextResult.UNREADABLE, result.context) @@ -123,9 +151,8 @@ def test_garbage_result_redeems_as_unreadable(self): def identifiers_for(self, name, value): """Asks the ``json`` endpoint for a 51Did with the given query - parameter and returns every identifier it answered with. An empty - list means the resource key is not entitled to that usage, which - the caller reports rather than fails.""" + parameter, and returns every identifier it answered with and the + ``fodid`` member, which holds the reason when there is none.""" url = "{0}{1}.json?{2}&values=FODiD.IdProbGlobal" \ "&values=FODiD.IdProbLic".format( self.client.endpoint, @@ -137,11 +164,12 @@ def identifiers_for(self, name, value): with urllib.request.urlopen(request, timeout=30) as response: body = json.loads(response.read().decode("utf-8")) fodid = body.get("fodid") or {} - return [ + identifiers = [ FodId.from_base64(fodid[field]) for field in ("idprobglobal", "idproblic") if fodid.get(field) ] + return identifiers, fodid def assert_aligned(self, label, fod_id, usage, terms, indirect): """Asserts the terms and every field the flags byte carries, read @@ -170,35 +198,23 @@ def test_every_usage_reads_back_the_terms_and_flags(self): ends at the match key and reads as no terms. """ cases = [ - ("non-marketing", Usage.NON_MARKETING, None), - ("standard", Usage.STANDARD, self.MODEL_TERMS_FOR_MARKETING_2), + ("non-marketing", Usage.NON_MARKETING, None, FODID), + ("standard", Usage.STANDARD, self.MODEL_TERMS_FOR_MARKETING_2, + MARKETING), ("personalized", Usage.PERSONALIZED, - self.MODEL_TERMS_FOR_MARKETING_2), + self.MODEL_TERMS_FOR_MARKETING_2, MARKETING), ] - checked = 0 - for name, usage, terms in cases: - identifiers = self.identifiers_for("id.usage", name) + for name, usage, terms, product in cases: + identifiers, fodid = self.identifiers_for("id.usage", name) if not identifiers: - print("id.usage={0}: no identifier returned, so this key is " - "not entitled to that usage.".format(name), - file=sys.stderr) - continue + self.fail(answered_without( + "a 51Did for id.usage=" + name, null_reason(fodid), + product)) for index, fod_id in enumerate(identifiers): self.assert_aligned( "{0}[{1}]".format(name, index), fod_id, usage, terms, False) - if terms is not None: - checked += len(identifiers) - - # Reported as a skip rather than printed, because a runner shows - # the counts and not the output, so a run that proved nothing would - # otherwise look identical to one that proved everything. - if checked == 0: - self.skipTest( - "this resource key returned no marketing 51Did, so no terms " - "address was read and this run did not prove it; use a key " - "entitled to the standard or personalized usage") def test_consent_string_sets_the_usage_is_indirect_bit(self): """A consent management platform sends an IAB TCF consent string @@ -219,39 +235,28 @@ def test_consent_string_sets_the_usage_is_indirect_bit(self): the Appendix 1 standard set of 1, 2, 7, 8 and 11. """ cases = [ - ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage.PERSONALIZED), - ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage.STANDARD), + ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage.PERSONALIZED, + "personalized"), + ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage.STANDARD, "standard"), ] - proven = 0 - for tc_string, usage in cases: + for tc_string, usage, name in cases: # No id.usage is sent. A stated usage wins over a consent # string, so sending one would leave the bit clear and this # would prove the opposite of what it says. - identifiers = self.identifiers_for("tcstring", tc_string) + identifiers, fodid = self.identifiers_for("tcstring", tc_string) if not identifiers: - print("consent string granting {0}: no identifier returned, " - "so this key is not entitled to that marketing " - "usage.".format(usage), file=sys.stderr) - continue + self.fail(answered_without( + "a 51Did for a consent string granting the " + name + + " usage", null_reason(fodid), MARKETING)) for index, fod_id in enumerate(identifiers): # A consent string granting a marketing usage produces a # marketing identifier, so the terms travel with it too. self.assert_aligned( - "consent/{0}[{1}]".format(usage, index), fod_id, usage, + "consent/{0}[{1}]".format(name, index), fod_id, usage, self.MODEL_TERMS_FOR_MARKETING_2, True) - proven += len(identifiers) - - # Same reasoning as the usage test above. - if proven == 0: - self.skipTest( - "this resource key returned no identifier for either consent " - "string, so the usage is indirect bit was never read and " - "this run did not prove it") - -@unittest.skipUnless( - RESOURCE_KEY, - "set resource_key (or _51DEGREES_RESOURCE_KEY) to run the live tests") + + class BrowserCreatorContextLiveTests(unittest.TestCase): """The whole browser path against the live service, as a page runs it. @@ -264,6 +269,10 @@ class BrowserCreatorContextLiveTests(unittest.TestCase): service's contract with every one of these packages, and a name the service adds, drops or renames has to be seen here rather than by a customer. + + Without a licence key the sealed result cannot be opened, so each test + then checks that it redeems as unreadable, which is what a caller with + no licence key gets. """ # A browser the service will judge as a browser, and a second one @@ -297,9 +306,10 @@ class BrowserCreatorContextLiveTests(unittest.TestCase): "transport", "device", "browserip", "connectionip", "asn", "platformname", "platformversion", "browsername", "browserversion") - @classmethod - def setUpClass(cls): - cls.client = DidClient(RESOURCE_KEY, LICENCE_KEY) + def setUp(self): + if not RESOURCE_KEY: + self.fail(NO_KEY) + self.client = DidClient(RESOURCE_KEY, LICENCE_KEY) def get(self, url, user_agent): request = urllib.request.Request( @@ -325,14 +335,10 @@ def create(self, usage="standard"): value = fodid.get("idproblic") or fodid.get("idprobglobal") if not value: # The probabilistic reason in full, because it names every - # snippet result the service is still waiting for, and a cut - # dump of the whole member hides them behind the email ones. - reason = fodid.get("idprobglobalnullreason") \ - or fodid.get("idproblicnullreason") \ - or json.dumps(fodid)[:300] - self.skipTest( - "this resource key returned no identifier for a browser " - "shaped request: " + reason) + # snippet result the service is still waiting for. + self.fail(answered_without( + "a 51Did for a browser shaped request with id.usage=" + + usage, null_reason(fodid), MARKETING)) return FodId.from_base64(value) def verify_context(self, fod_id, user_agent): @@ -346,13 +352,16 @@ def verify_context(self, fod_id, user_agent): urllib.parse.urlencode(query)) status, body = self.get(url, user_agent) if status == 404: - self.skipTest("the host does not offer the creator context") + self.fail(no_creator_context( + self.client.endpoint, + "answered 404 from id/verify-context")) self.assertEqual(200, status, body[:300]) answer = json.loads(body) if "result" not in answer: - self.skipTest( - "this host holds no context secret, so it answered in the " - "open with " + body[:200]) + self.fail(no_creator_context( + self.client.endpoint, + "holds no creator context secret, so it answered in the " + "open with " + body[:200])) return answer["result"] def test_a_page_creates_reads_back_verifies_and_redeems(self): @@ -364,19 +373,18 @@ def test_a_page_creates_reads_back_verifies_and_redeems(self): result = self.verify_context(fod_id, self.CHROME) redeemed = run(self.client.redeem(fod_id, result)) if not self.client.has_licence_key: - # Without a licence key the sealed result cannot be opened, - # which the service reports as unreadable. self.assertEqual(ContextResult.UNREADABLE, redeemed.context) - self.skipTest("set license_key to redeem a sealed result") + return self.assertEqual(ContextResult.VERIFIED, redeemed.context) self.assertEqual("verified", redeemed.signature.value) def test_another_browser_redeems_as_a_mismatch_naming_every_factor(self): - if not LICENCE_KEY: - self.skipTest("set license_key to redeem a sealed result") fod_id = self.create() result = self.verify_context(fod_id, self.SAFARI) redeemed = run(self.client.redeem(fod_id, result)) + if not self.client.has_licence_key: + self.assertEqual(ContextResult.UNREADABLE, redeemed.context) + return self.assertEqual(ContextResult.MISMATCH, redeemed.context) self.assertEqual("verified", redeemed.signature.value, "the identifier is genuine whichever browser " diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py new file mode 100644 index 00000000..e52b5f98 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py @@ -0,0 +1,113 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""What a key on the Basic tier is entitled to, asked of the live cloud. + +A Basic key carries the FODid product, so it gets a 51Did for the +non-marketing usage. It does not carry the CloudV5FODiD product, so the +service refuses it the standard and personalized usages and says which +product is missing. The key is read from ``_51DEGREES_RESOURCE_KEY_BASIC`` +only, because these tests assert what that one tier gets, and they fail +when it is not set. +""" + +import json +import urllib.parse +import urllib.request +import unittest + +import pytest + +from fiftyone_pipeline_did import DidClient, FodId, Usage +from fiftyone_pipeline_did.did_client import USER_AGENT + +from .live_cloud import answered_without, find_resource_key, no_resource_key + +pytestmark = [pytest.mark.live, pytest.mark.tier] + +BASIC_KEY_NAMES = ("_51DEGREES_RESOURCE_KEY_BASIC",) +BASIC_KEY = find_resource_key(BASIC_KEY_NAMES) + +FIELDS = ("idprobglobal", "idproblic") + + +class BasicTierLiveTests(unittest.TestCase): + + def setUp(self): + if not BASIC_KEY: + self.fail(no_resource_key( + "the FODid product on the Basic tier", + "a 51Did under each id.usage, expecting the marketing " + "usages to be refused", several=True, + names=BASIC_KEY_NAMES)) + self.endpoint = DidClient(BASIC_KEY).endpoint + + def fodid_for(self, usage): + """The ``fodid`` member of the ``json`` endpoint's answer for the + usage.""" + url = "{0}{1}.json?{2}".format( + self.endpoint, urllib.parse.quote(BASIC_KEY, safe=""), + urllib.parse.urlencode([ + ("id.usage", usage), + ("values", "FODiD.IdProbGlobal"), + ("values", "FODiD.IdProbLic")])) + request = urllib.request.Request( + url, data=b"", headers={"User-Agent": USER_AGENT}, + method="POST") + with urllib.request.urlopen(request, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + return body.get("fodid") or {} + + def test_non_marketing_usage_gets_a_51did(self): + fodid = self.fodid_for("non-marketing") + for field in FIELDS: + if not fodid.get(field): + self.fail(answered_without( + field + " for id.usage=non-marketing", + fodid.get(field + "nullreason"), "the FODid product")) + self.assertEqual( + Usage.NON_MARKETING, FodId.from_base64(fodid[field]).usage, + field) + + def test_marketing_usages_are_refused_naming_the_product(self): + for usage in ("standard", "personalized"): + fodid = self.fodid_for(usage) + for field in FIELDS: + self.assertFalse( + fodid.get(field), + "A Basic key got {0} for id.usage={1}, which only a key " + "that carries the CloudV5FODiD product should get. Check " + "the key is still on the Basic tier.".format( + field, usage)) + name = "IdProbGlobal" if field == "idprobglobal" \ + else "IdProbLic" + self.assertEqual( + "Can not form {0}. For marketing (personalized or " + "standard) usages a CloudV5FODiD product needs to be a " + "part of the license.".format(name), + fodid.get(field + "nullreason"), + "the reason given for {0} with id.usage={1}".format( + field, usage)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tests/test_fodid.py b/pipeline/fiftyone_pipeline_did/tests/test_fodid.py index 3b6b445d..dad12a7a 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_fodid.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_fodid.py @@ -852,6 +852,168 @@ def test_raising_and_non_raising_readers_agree_on_success(self): self.assertEqual(raising.match_key, result.value.match_key) +#: Every character with the Unicode White_Space property, which is what the +#: 51Did specification means by white space. +WHITE_SPACE = ( + "\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000") + +#: Characters ``str.strip()`` removes or that look like white space, but +#: which do not have the White_Space property, so they are outside the +#: alphabet wherever they sit. U+001C to U+001F are the ones Python's own +#: ``str.isspace()`` takes, and U+FEFF and U+200B are zero width +#: characters that cannot be seen in the text. +NOT_WHITE_SPACE = "\x1c\x1d\x1e\x1f\ufeff\u200b" + + +def _forms(standard): + """The standard text, and the URL-safe text with and without its + padding.""" + url_safe = standard.replace("+", "-").replace("/", "_") + return (standard, url_safe, url_safe.rstrip("=")) + + +def _spaced(text, space, count): + """The text with the white space put in at a number of points spread + evenly through it.""" + spaced = text + for point in range(count, 0, -1): + at = len(text) * point // (count + 1) + spaced = spaced[:at] + space + spaced[at:] + return spaced + + +def _show(text): + """The text with each character outside printable ASCII written as its + code, so a failure message shows where it is.""" + return "".join( + c if " " < c <= "~" else "\\u{0:04X}".format(ord(c)) for c in text) + + +class FodIdBase64TextTests(unittest.TestCase): + """How the base64 readers treat the text around and inside a 51Did, as + the identifier layout page of the 51Did specification sets out. White + space before and after the text is ignored, any other character outside + the alphabet is refused, white space inside the text included, and a + text holding a character from each alphabet is refused.""" + + def setUp(self): + self.factory = FodIdTestFactory() + self.standard = self._standard_with_both_specials() + + def _standard_with_both_specials(self): + """The standard text of a real signed 51Did holding both ``+`` and + ``/``, so every form below holds both characters of its alphabet. + The signature differs on every signing, so a few tries find one.""" + for _ in range(100): + standard = self.factory.signed_owid_base64(canonical_payload()) + if "+" in standard and "/" in standard: + return standard + self.fail("No signed envelope contained both + and / in 100 tries.") + + def assert_reads_as_standard(self, text): + result = FodId.try_from_base64(text) + self.assertTrue(result.ok, "{0} for {1}".format( + result.status.value, _show(text))) + self.assertEqual(self.standard, result.value.as_base64()) + self.assertEqual( + self.standard, FodId.from_base64(text).as_base64()) + + def assert_invalid_base64(self, text): + """Both readers refuse the text as invalid base64, and the step + they share gives nothing for it.""" + result = FodId.try_from_base64(text) + self.assertFalse(result.ok, "Parsed {0}".format(_show(text))) + self.assertIsNone(result.value) + self.assertIs(FodIdParseStatus.INVALID_BASE64, result.status, + _show(text)) + with self.assertRaises(OwidError) as raised: + FodId.from_base64(text) + self.assertIn("InvalidBase64", str(raised.exception)) + self.assertIsNone(FodId.to_standard_base64(text), _show(text)) + + def test_the_text_holds_both_specials_in_every_form(self): + standard, url_safe, unpadded = _forms(self.standard) + self.assertTrue("+" in standard and "/" in standard) + self.assertTrue("-" in url_safe and "_" in url_safe) + self.assertTrue("-" in unpadded and "_" in unpadded) + self.assertNotIn("=", unpadded) + + def test_every_form_parses(self): + for text in _forms(self.standard): + with self.subTest(text=text): + self.assert_reads_as_standard(text) + + def test_white_space_around_the_text_parses(self): + for before, after in ((" ", "\n"), ("\t", "\r\n"), + ("\u00a0", "\u00a0"), ("\u3000", "\u3000")): + for text in _forms(self.standard): + with self.subTest(before=_show(before), text=text): + self.assert_reads_as_standard(before + text + after) + + def test_every_white_space_character_around_the_text_parses(self): + self.assertEqual(25, len(WHITE_SPACE)) + for space in WHITE_SPACE: + for text in _forms(self.standard): + with self.subTest(space=_show(space), text=text): + self.assert_reads_as_standard(space + text + space) + + def test_white_space_inside_the_text_is_invalid_base64(self): + for space, count in ((" ", 1), (" ", 2), (" ", 3), ("\t", 4), + ("\r\n", 2), ("\u00a0", 4), ("\u3000", 1)): + for text in _forms(self.standard): + with self.subTest(space=_show(space), count=count, + text=text): + self.assert_invalid_base64(_spaced(text, space, count)) + + def test_every_white_space_character_inside_the_text_is_refused(self): + for space in WHITE_SPACE: + for text in _forms(self.standard): + with self.subTest(space=_show(space), text=text): + self.assert_invalid_base64(_spaced(text, space, 1)) + + def test_characters_that_are_not_white_space_are_refused_anywhere(self): + # Python's own strip() removes U+001C to U+001F, so they are + # checked around the text as well as inside it. + for character in NOT_WHITE_SPACE: + for text in _forms(self.standard): + for spaced in (character + text, text + character, + _spaced(text, character, 1)): + with self.subTest(text=_show(spaced)): + self.assert_invalid_base64(spaced) + + def test_both_alphabets_is_invalid_base64(self): + url_safe = FodId.to_base64_url(self.standard) + for text, from_, to in ((self.standard, "+", "-"), + (self.standard, "/", "_"), + (url_safe, "-", "+"), + (url_safe, "_", "/")): + with self.subTest(from_=from_, to=to): + at = text.index(from_) + self.assert_invalid_base64(text[:at] + to + text[at + 1:]) + + def test_white_space_alone_is_still_missing_input(self): + for text in (" ", "\t\r\n", "\u00a0", "\u3000", WHITE_SPACE): + with self.subTest(text=_show(text)): + result = FodId.try_from_base64(text) + self.assertFalse(result.ok) + self.assertIsNone(result.value) + self.assertIs(FodIdParseStatus.MISSING_INPUT, result.status) + + def test_to_standard_base64_restores_alphabet_and_padding(self): + for value, expected in (("-_", "+/=="), ("+/==", "+/=="), + ("", ""), (" YQ\n", "YQ=="), + ("\u3000-_\u3000", "+/==")): + with self.subTest(value=_show(value)): + self.assertEqual(expected, FodId.to_standard_base64(value)) + + def test_to_standard_base64_gives_none_for_a_text_it_refuses(self): + for value in ("YW Jj", "YW\tJj", "YW\r\nJj", "YW\u00a0Jj", + "YWJj\x1c", "+_", "-/", "+-", "/_"): + with self.subTest(value=_show(value)): + self.assertIsNone(FodId.to_standard_base64(value)) + + class FodIdTermsTests(unittest.TestCase): """The Terms byte, which says which terms document the identifier was created under so that the terms travel with the identifier. The byte is diff --git a/pipeline/fiftyone_pipeline_did/tox.ini b/pipeline/fiftyone_pipeline_did/tox.ini index d80d881b..edf34540 100644 --- a/pipeline/fiftyone_pipeline_did/tox.ini +++ b/pipeline/fiftyone_pipeline_did/tox.ini @@ -12,13 +12,17 @@ package = wheel wheel_build_env = .pkg # tox runs the tests in a clean environment and strips anything it is not # told to keep, so without this the live cloud tests never see the resource -# key and skip themselves on every run, which reads as a pass. +# key and fail on every run. PYTEST_ADDOPTS carries the markers the CI +# scripts select with. passenv = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE + _51DEGREES_RESOURCE_KEY_BASIC _51DEGREES_LICENSE_KEY resource_key license_key FOD_CLOUD_API_URL + PYTEST_ADDOPTS deps = {[common]deps} # owid is provided by the owid-python submodule (../owid-python), not diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json index 90cfbada..7196a8cb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json @@ -9,8 +9,10 @@ "interval": 1200, "requested_package_size": 50, "cookie": "", + "track_session": false, "query_whitelist": [], - "header_blacklist": [] + "header_blacklist": [], + "share_all": false } } ] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 3173318e..210dfb0a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -31,7 +31,8 @@ import requests from fiftyone_pipeline_engines.engine import Engine -from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter +from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ + ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker # The maximum length of a piece of evidence's value which can be @@ -54,7 +55,9 @@ def __init__( header_blacklist = [], share_percentage = 1, endpoint = "https://devices-v4.51degrees.com/new.ashx", - request_timeout=REQUEST_TIMEOUT): + request_timeout=REQUEST_TIMEOUT, + share_all = False, + track_session = False): """! Constructor for ShareUsage element @@ -70,22 +73,32 @@ def __init__( reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string - @param cookie: If a cookie is being used to identify user - sessions, it can be specified here in order to reduce the - sharing of duplicated data. + @param cookie: The name of the cookie that holds the session id. + It has no effect unless track_session is True. The cookie itself + is not shared unless share_all is True. + @type track_session: bool + @param track_session: If True, the cookie named by cookie is used + to tell user sessions apart, so evidence already shared is shared + again when it comes from a new session. False by default. It + needs cookie to be set, and has no effect when share_all is True, + because every cookie is then shared and used to find repeats. @type query_whitelist: list - @param query_whitelist: By default query string and HTTP form + @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here. - @type query_blacklist: list - @param query_blacklist: By default, all HTTP headers - (except a few, such as Cookies) are shared. - Individual headers can be excluded from sharing by adding them + If you need to share query string parameters, a list can be + specified here. None shares every parameter. + @type header_blacklist: list + @param header_blacklist: By default, all HTTP headers + except Cookie are shared. + Individual headers can be excluded from sharing by adding them to this list. @type share_percentage : float @param share_percentage: approximate proportion of requests to be shared. 1 = 100%, 0.5 = 50%, etc.. + @type share_all: bool + @param share_all: If True, all evidence is shared apart from the + email address, the salt used for hashed email identifiers, and the + request's query string and path. False by default. """ super(ShareUsage, self).__init__() @@ -102,6 +115,25 @@ def __init__( self.share_percentage = share_percentage self.endpoint = endpoint self.request_timeout = request_timeout + self.share_all = share_all + self.track_session = track_session + + # The session id and sequence number are kept in evidence, as the + # usage data carries them separately. + included = None if query_whitelist is None else \ + list(query_whitelist) + ["session-id", "sequence"] + + # When sessions are tracked, the pipeline keeps the session cookie + # for the tracker. The data sent leaves it out unless all evidence + # is shared. + self.evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, share_all) + self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, False, share_all) + self.tracker_evidence_key_filter = \ + ShareUsageTrackerEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, + share_all) # Add the share usage tracker which detects when # to send up sharing data @@ -188,11 +220,7 @@ def get_evidence_key_filter(self): """ - return ShareUsageEvidenceKeyFilter( - cookie = self.cookie, - query_whitelist = self.query_whitelist, - header_blacklist = self.header_blacklist - ) + return self.evidence_key_filter def share_send_usage(self): """! @@ -291,7 +319,7 @@ def process_internal(self, flow_data): """ if random.uniform(0, 1) <= self.share_percentage: - cachekey = self.get_evidence_key_filter().filter_evidence(flow_data.evidence.get_all()) + cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all()) cachekey = json.dumps(cachekey) share = self.tracker.track(cachekey) if share: @@ -327,7 +355,7 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.get_evidence_key_filter().filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key(key): data.try_add(key, value) return data @@ -394,7 +422,7 @@ def try_add(self, key, value): category = "" field = key - first_separator = key.index(".") + first_separator = key.find(".") if first_separator > 0: category = key[0:first_separator] field = key[first_separator + 1:len(key)] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index a9d0d2ac..5f2292f3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -22,41 +22,64 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +# Cookies, session values and query string parameters whose names start with +# this are the ones 51Degrees sets, and are shared. +FIFTYONE_PREFIX = "51d_" + +# The Cookie header carries every cookie, including the ones the cookie rule +# leaves out, so it is not shared unless all evidence is. +BLOCKED_HEADERS = ("cookie",) + +# Evidence that is never shared, whatever the settings, matched as the whole +# key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the +# salt used to make hashed email identifiers. 'request-query' is the whole query +# string, which would undo the rules for its parameters. 'request-path' would +# make each page a visitor opens look like a new session to the tracker. +NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path") + class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ - The ShareUsageEvidenceKeyFilter filters out all evidence - not needed by the 51Degrees ShareUsage service. - It allows for a specific whitelist of query strings, - a blacklist of headers and a specific cookie used for - session information + The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees + ShareUsage service is sent. Headers are shared unless they are blocked, + cookies and session values only when their names start with 51D_, and + query string and form parameters only when their names start with 51D_ or + are on the list to share. Any other evidence is shared. """ - def __init__(self, cookie = None, query_whitelist = [], header_blacklist = []): + def __init__( + self, + cookie = None, + query_whitelist = (), + header_blacklist = (), + include_session = False, + share_all = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string - @param cookie: which cookie is used to track evidence + @param cookie: the name of the cookie that identifies the session @type query_whitelist: list - @param query_whitelist: list of query string whitelist evidence to keep - @type query_blacklist: list - @param query_blacklist: list of header evidence to exclude - @type - @param + @param query_whitelist: query string and form parameters to share as + well as those starting 51D_, or None to share every parameter + @type header_blacklist: list + @param header_blacklist: headers not to share + @type include_session: bool + @param include_session: whether the session cookie is included + @type share_all: bool + @param share_all: whether all evidence is shared, apart from the + evidence that is never shared """ - self.query_whitelist = query_whitelist + self.cookie = cookie.lower() if cookie else None - self.header_blacklist = header_blacklist + self.query_whitelist = None if query_whitelist is None else \ + [parameter.lower() for parameter in query_whitelist] - self.cookie = cookie + self.header_blacklist = \ + [header.lower() for header in header_blacklist or ()] - if cookie is None and \ - (query_whitelist is None or len(query_whitelist) == 0) and \ - (header_blacklist is None or len(header_blacklist) == 0): - self.share_all = True - else: - self.share_all = False + self.include_session = include_session + + self.share_all = share_all - def filter_evidence_key(self, key): """! @@ -69,42 +92,58 @@ def filter_evidence_key(self, key): @return: Is this key in the filter's keys list? """ - # get prefix and key of evidence - key_parts = key.lower().split(".") - - prefix = key_parts[0] - suffix = key_parts[1] - - result = self.share_all - - if not self.share_all: - if prefix == "header": - # Add the header to the list if the header name does not - # appear in the list of blocked headers - result = suffix not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - result = suffix.startswith("51d_") or \ - (self.include_session and suffix == self.cookie) - elif prefix == "session": - # Only session values that start with the 51Degrees - # cookie prefix - result = suffix.startswith("51d_") - elif prefix == "query": - # If no query string parameter filter was specified - # then share all of them. - # Otherwise, only include query string parameters that - # start with 51d_ or that have been specified in - # the constructor. - result = self.query_whitelist is None or \ - len(self.query_whitelist) == 0 or \ - suffix.startswith("51d_") or \ - suffix in self.query_whitelist - else: - # Add anything that is not a cookie, header, session - # variable or query parameter - result = True - - return result - \ No newline at end of file + key = key.lower() + + if any(key == never_shared or key.endswith("." + never_shared) + for never_shared in NEVER_SHARED_KEYS): + return False + + if self.share_all: + return True + + # The prefix is everything before the first dot and the field + # everything after it, so a field can itself contain dots. + prefix, separator, field = key.partition(".") + + if not separator: + return True + elif prefix == "header": + # Add the header to the list if the header name does not + # appear in the list of blocked headers + return field not in self.header_blacklist and \ + field not in BLOCKED_HEADERS + elif prefix == "cookie": + # Only add cookies that start with the 51Degrees cookie + # prefix + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + elif prefix == "session": + # Only session values that start with the 51Degrees + # cookie prefix + return field.startswith(FIFTYONE_PREFIX) + elif prefix == "query": + # If no query string parameter list was given then share all + # of them. Otherwise, only include query string parameters + # that start with 51d_ or that are in the list. + return self.query_whitelist is None or \ + field.startswith(FIFTYONE_PREFIX) or \ + field in self.query_whitelist + else: + # Add anything that is not a cookie, header, session + # variable or query parameter + return True + +class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): + """ + The evidence the ShareUsageTracker compares to decide whether a request + repeats one already shared. The session id and sequence number are left + out, because they make every request different. + """ + + def filter_evidence_key(self, key): + + if key.lower() in ("query.session-id", "query.sequence"): + return False + + return super(ShareUsageTrackerEvidenceKeyFilter, self)\ + .filter_evidence_key(key) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py new file mode 100644 index 00000000..16184d52 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -0,0 +1,168 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ + ShareUsageEvidenceKeyFilter + + +class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): + + def test_default_rules(self): + """! + Check that by default headers other than Cookie are shared, cookies, + session values and query string parameters only when their names + start with 51D_, and any other evidence. + """ + + keys = ShareUsageEvidenceKeyFilter() + + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + self.assertFalse(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.51d_screenpixelswidth")) + self.assertTrue(keys.filter_evidence_key("cookie.51D_ScreenPixelsWidth")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.a.b")) + self.assertTrue(keys.filter_evidence_key("session.51d_value")) + self.assertFalse(keys.filter_evidence_key("session.org.example.attribute")) + self.assertTrue(keys.filter_evidence_key("query.51d_gethighentropyvalues")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertTrue(keys.filter_evidence_key("server.client-ip")) + self.assertTrue(keys.filter_evidence_key("nodot")) + + def test_field_is_everything_after_the_first_dot(self): + """! + Check that a name containing a dot is matched as a whole against + the parameters to share. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertFalse(keys.filter_evidence_key("query.valid")) + self.assertFalse(keys.filter_evidence_key("query.other.name")) + + def test_lists_ignore_case(self): + """! + Check that the headers not to share and the parameters to share + are matched without regard to case. + """ + + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=["Campaign"], + header_blacklist=["X-Secret"]) + + self.assertFalse(keys.filter_evidence_key("header.x-secret")) + self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + + def test_no_parameter_list_shares_every_parameter(self): + """! + Check that None in place of the parameters to share shares every + parameter, and an empty list only those starting 51D_. + """ + + every = ShareUsageEvidenceKeyFilter(query_whitelist=None) + prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + + self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(prefixed.filter_evidence_key("query.campaign")) + self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + + def test_session_cookie(self): + """! + Check that the session cookie is included only when asked for, and + that no other ordinary cookie is. + """ + + kept = ShareUsageEvidenceKeyFilter(cookie="SessionId", include_session=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId") + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + self.assertFalse(sent.filter_evidence_key("cookie._ga")) + + def test_share_all(self): + """! + Check that sharing everything shares the evidence the other rules + leave out, apart from the evidence that is never shared. + """ + + keys = ShareUsageEvidenceKeyFilter(share_all=True) + + self.assertTrue(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue(keys.filter_evidence_key("query.password")) + self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("server.request-method")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertFalse(keys.filter_evidence_key("header.id.email")) + self.assertFalse(keys.filter_evidence_key("query.id.salt")) + self.assertFalse(keys.filter_evidence_key("header.id.salt")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertFalse(keys.filter_evidence_key("server.request-path")) + + def test_never_shared_in_every_mode(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are never shared, + whether bare or after a prefix, and without regard to case. + """ + + for keys in ( + ShareUsageEvidenceKeyFilter(), + ShareUsageEvidenceKeyFilter(query_whitelist=None), + ShareUsageEvidenceKeyFilter(share_all=True)): + for key in ( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"): + self.assertFalse(keys.filter_evidence_key(key), key) + + def test_never_shared_keys_match_whole_parts(self): + """! + Check that a key that only ends with the same letters as a key + never shared is treated as an ordinary key. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue( + ShareUsageEvidenceKeyFilter(share_all=True) + .filter_evidence_key("query.valid.email")) + + def test_naming_a_parameter_does_not_share_the_query_string(self): + """! + Check that sharing one parameter does not share the whole query + string, which carries every parameter. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + + self.assertTrue(keys.filter_evidence_key("query.utm_source")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 8329645b..4f1bea40 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -431,6 +431,302 @@ def test_includes_header(self): self.assertIn('&', app.received[0]) + def test_default_shares_only_accepted_evidence(self): + """! + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") + flowdata.evidence.add("query.51d_gethighentropyvalues", "hev") + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("header.cookie", "sessionid=header value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('
some user agent
', app.received[0]) + self.assertIn('1080', app.received[0]) + self.assertIn('hev', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertNotIn("header value", app.received[0]) + self.assertNotIn("password value", app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_never_shared_evidence(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are not shared, though + the rules for their prefixes would share them. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("id.email", "email value") + flowdata.evidence.add("header.id.salt", "salt value") + flowdata.evidence.add("server.request-query", "query string value") + flowdata.evidence.add("server.request-path", "path value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("email value", app.received[0]) + self.assertNotIn("salt value", app.received[0]) + self.assertNotIn("query string value", app.received[0]) + self.assertNotIn("path value", app.received[0]) + + def test_share_all(self): + """! + Check that sharing all evidence shares what the other rules leave + out, but not the evidence that is never shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + share_all=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('cookie value', app.received[0]) + self.assertIn('campaign value', app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_parameter_names_with_dots(self): + """! + Check that a query string parameter whose name contains a dot is + matched by its whole name against the parameters to share. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=["valid.email"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.valid.email", "valid value") + flowdata.evidence.add("query.valid", "first part value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('valid value', app.received[0]) + self.assertNotIn("first part value", app.received[0]) + + def test_empty_query_whitelist(self): + """! + Check that an empty list of parameters to share shares only those + starting 51D_, and that None shares every parameter. + """ + + for query_whitelist, shared in (([], False), (None, True)): + app.received = [] + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=query_whitelist, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.51d_param", "fiftyone value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("fiftyone value", app.received[0]) + self.assertEqual("campaign value" in app.received[0], shared) + + def test_session_cookie(self): + """! + Check that with session tracking on the session cookie is kept in + the evidence, but neither it nor any other ordinary cookie is + shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.sessionid", "session value") + flowdata.evidence.add("cookie._ga", "analytics value") + flowdata.process() + + self.waitForUsageThreads() + self.assertTrue(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("session value", app.received[0]) + self.assertNotIn("analytics value", app.received[0]) + + def test_session_cookie_without_session_tracking(self): + """! + Check that session tracking is off by default, and that naming the + session cookie then has no effect. The cookie is not kept, and + evidence already shared is not shared again for a new session. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid") + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertNotIn("session value", app.received[0]) + self.assertFalse(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertFalse(usage.track_session) + + def test_session_tracking(self): + """! + Check that with session tracking on, evidence already shared is + shared again for a new session but not for a repeat in the same + session, and that the session cookie is not shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 2) + received = "\n".join(app.received) + self.assertEqual(received.count("repeated user agent"), 2) + self.assertNotIn("session value", received) + + def test_ordinary_cookie_with_header_blacklist(self): + """! + Check that an ordinary cookie is left out without an error when + some headers are not to be shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("header.x-secret", "secret value") + flowdata.evidence.add("cookie.csrftoken", "token value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("secret value", app.received[0]) + self.assertNotIn("token value", app.received[0]) + + def test_repeat_with_new_sequence_not_shared(self): + """! + Check that a request differing from one already shared only in + its session id and sequence number is not shared again. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session, sequence in (("session 1", "1"), ("session 2", "2")): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("query.session-id", session) + flowdata.evidence.add("query.sequence", sequence) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("session 1", app.received[0]) + + def test_evidence_without_a_prefix(self): + """! + Check that evidence whose key has no prefix is shared without an + error. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("nodot", "no prefix value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("no prefix value", app.received[0]) + # zzz is to make this test run last so no other tests depend on the receiver anymore # we depend on python unittests ordering tests alphabetically, which is not ideal, but works for now def test_zzz_timeout(self): diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py index 794ec987..619f843b 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import warnings from enum import Enum @@ -38,3 +39,56 @@ class MissingTranslationBehavior(Enum): # Add a not-found error to the flow data and return None. FLOW_ERROR = "flow_error" + + @classmethod + def from_value(cls, value): + + """! + The member a value names. A member is returned as it is. Text names + a member by the specification's name (Original, EmptyString or + FlowError) or by the member's own name or value, ignoring case and + underscores, because a configuration file cannot give a member. + + @param value: A member, or text naming one. + + @rtype: MissingTranslationBehavior + @return: The member named. + + @raises ValueError: When the value names no member. + """ + + if isinstance(value, cls): + return value + if isinstance(value, str): + key = value.replace("_", "").lower() + for member in cls: + if member.name.replace("_", "").lower() == key: + return member + raise ValueError( + f"{value!r} is not a missing translation behavior. Use " + f"Original, EmptyString or FlowError.") + + +def _resolve(missing_translation_behavior, behavior): + + """! + The member the constructor arguments name. None means not given, as for + the other engines' optional settings, so a setting passed on as None + gives ORIGINAL. behavior is the deprecated name for + missing_translation_behavior, so using it warns, and when both are given + missing_translation_behavior wins. Neither given means ORIGINAL. + """ + + if behavior is not None: + # Two levels up is the caller of the constructor that called this. + warnings.warn( + "The 'behavior' argument is deprecated. Use " + "'missing_translation_behavior', the name the specification " + "gives the option.", + DeprecationWarning, + stacklevel=3) + if missing_translation_behavior is None: + missing_translation_behavior = behavior + if missing_translation_behavior is None: + return MissingTranslationBehavior.ORIGINAL + return MissingTranslationBehavior.from_value(missing_translation_behavior) diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py index 4f349e76..8be1d175 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py @@ -31,7 +31,8 @@ from .languages import Languages, validate_locale from .translator import Translator -from .missing_translation_behavior import MissingTranslationBehavior +from .missing_translation_behavior import ( + MissingTranslationBehavior, _resolve) class TranslationEngineBase(FlowElement): @@ -65,7 +66,9 @@ def __init__( translations, sources, fixed_language=None, - behavior=MissingTranslationBehavior.ORIGINAL): + missing_translation_behavior=None, + *, + behavior=None): """! Constructor. @@ -86,8 +89,17 @@ def __init__( always translates to this language, otherwise the language is resolved from the evidence. - @type behavior: MissingTranslationBehavior - @param behavior: Behavior when a translation is missing for a value. + @type missing_translation_behavior: MissingTranslationBehavior or + string + @param missing_translation_behavior: Behavior when a translation is + missing for a value, as a member or as text naming one, such as + "EmptyString". ORIGINAL when not given or None. Any other value + raises ValueError. + + @type behavior: MissingTranslationBehavior or string + @param behavior: Deprecated name for missing_translation_behavior, + which raises a DeprecationWarning when used. When both are given, + missing_translation_behavior wins. """ super().__init__() @@ -100,6 +112,8 @@ def __init__( if not source_element_data_key or not source_element_data_key.strip(): raise ValueError("The source element key must be configured.") + behavior = _resolve(missing_translation_behavior, behavior) + self.datakey = "translation" self.source_element_data_key = source_element_data_key.strip() self._behavior = behavior diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py index 19be2cd3..0a014c25 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py @@ -22,7 +22,8 @@ from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from .missing_translation_behavior import MissingTranslationBehavior +from .missing_translation_behavior import ( + MissingTranslationBehavior, _resolve) class Translator: @@ -41,7 +42,9 @@ class Translator: def __init__( self, translations=None, - behavior=MissingTranslationBehavior.ORIGINAL): + missing_translation_behavior=None, + *, + behavior=None): """! Constructor. @@ -50,15 +53,24 @@ def __init__( @param translations: Map of source value to translated value. Lookups are case-insensitive. - @type behavior: MissingTranslationBehavior - @param behavior: Behavior when a translation is missing for a value. + @type missing_translation_behavior: MissingTranslationBehavior or + string + @param missing_translation_behavior: Behavior when a translation is + missing for a value, as a member or as text naming one, such as + "EmptyString". ORIGINAL when not given or None. Any other value + raises ValueError. + + @type behavior: MissingTranslationBehavior or string + @param behavior: Deprecated name for missing_translation_behavior, + which raises a DeprecationWarning when used. When both are given, + missing_translation_behavior wins. """ translations = translations or {} # Store with lower-cased keys so lookups are case-insensitive. self._translations = { str(key).lower(): value for key, value in translations.items()} - self._behavior = behavior + self._behavior = _resolve(missing_translation_behavior, behavior) def translate(self, value, errors=None): diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_translation.py b/pipeline/fiftyone_pipeline_translation/tests/test_translation.py index ebafc0d9..644e23b9 100644 --- a/pipeline/fiftyone_pipeline_translation/tests/test_translation.py +++ b/pipeline/fiftyone_pipeline_translation/tests/test_translation.py @@ -21,6 +21,7 @@ # ********************************************************************* import unittest +import warnings from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary @@ -183,5 +184,269 @@ def test_locale_from_filename(self): TranslationEngineBase._get_language_name("countries.fr_FR.yml")) +# Each text the missing translation behavior accepts, and the member it +# names. The specification's names, the members' names and values, and +# other mixes of case and underscores are all accepted. +BEHAVIOR_TEXTS = { + "Original": MissingTranslationBehavior.ORIGINAL, + "original": MissingTranslationBehavior.ORIGINAL, + "ORIGINAL": MissingTranslationBehavior.ORIGINAL, + "EmptyString": MissingTranslationBehavior.EMPTY_STRING, + "emptystring": MissingTranslationBehavior.EMPTY_STRING, + "empty_string": MissingTranslationBehavior.EMPTY_STRING, + "EMPTY_STRING": MissingTranslationBehavior.EMPTY_STRING, + "Empty_String": MissingTranslationBehavior.EMPTY_STRING, + "FlowError": MissingTranslationBehavior.FLOW_ERROR, + "flowerror": MissingTranslationBehavior.FLOW_ERROR, + "flow_error": MissingTranslationBehavior.FLOW_ERROR, + "FLOW_ERROR": MissingTranslationBehavior.FLOW_ERROR, +} + +# Values that name no member, each refused when the engine is built. +UNKNOWN_BEHAVIORS = ( + "", "Empty String", "Origin", "Missing", "flow-error", 1, + ["Original"]) + +# The specification's name for each member. +SPECIFICATION_NAMES = { + "Original": MissingTranslationBehavior.ORIGINAL, + "EmptyString": MissingTranslationBehavior.EMPTY_STRING, + "FlowError": MissingTranslationBehavior.FLOW_ERROR, +} + +# The value a missing translation of "Spain" gives under each member, with +# the number of flow errors it adds. +MISSING_RESULTS = { + MissingTranslationBehavior.ORIGINAL: (["Allemagne", "Spain"], 0), + MissingTranslationBehavior.EMPTY_STRING: (["Allemagne", ""], 0), + MissingTranslationBehavior.FLOW_ERROR: (["Allemagne", None], 1), +} + + +class MissingTranslationBehaviorTests(unittest.TestCase): + + """ + The missing translation behavior given as a member or as text, as a + configuration file gives it, with anything else refused when the engine + is built, apart from None, which means not given. The option takes the + specification's name, missing_translation_behavior, and still takes the + old name, behavior, with a DeprecationWarning. + """ + + def _engine(self, behavior): + return TranslationEngineBase( + "source", + [("Names", "NamesTranslated")], + FR_SOURCE, + "fr_FR", + behavior) + + def _missing(self, pipeline): + """The translated names and the flow errors for a source holding a + name with a translation and one without. The pipeline keeps errors + on the flow data rather than raising them, so a flow error can be + counted.""" + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.process() + translated = flowdata.get("translation").get("NamesTranslated") + if isinstance(translated, AspectPropertyValue): + translated = translated.value() + return translated, len(flowdata.errors) + + def _pipeline(self, behavior): + return self._pipeline_with(self._engine(behavior)) + + @staticmethod + def _pipeline_with(engine): + source = MockSourceElement("source", { + "Names": AspectPropertyValue(value=["Germany", "Spain"]) + }) + return PipelineBuilder({"suppress_process_exceptions": True}).add( + source).add(engine).build() + + @staticmethod + def _engine_with(**options): + """An engine built with every option given by its name.""" + return TranslationEngineBase( + source_element_data_key="source", + translations=[("Names", "NamesTranslated")], + sources=FR_SOURCE, + fixed_language="fr_FR", + **options) + + def test_from_value_names_each_member(self): + for text, member in BEHAVIOR_TEXTS.items(): + with self.subTest(text): + self.assertIs( + member, MissingTranslationBehavior.from_value(text)) + + def test_from_value_returns_a_member_as_it_is(self): + for member in MissingTranslationBehavior: + with self.subTest(member.name): + self.assertIs( + member, MissingTranslationBehavior.from_value(member)) + + def test_each_text_takes_effect(self): + for text, member in BEHAVIOR_TEXTS.items(): + with self.subTest(text): + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline(text))) + + def test_each_member_takes_effect(self): + for member, expected in MISSING_RESULTS.items(): + with self.subTest(member.name): + self.assertEqual( + expected, self._missing(self._pipeline(member))) + + def test_unknown_value_is_refused_when_the_engine_is_built(self): + for value in UNKNOWN_BEHAVIORS: + with self.subTest(repr(value)): + with self.assertRaises(ValueError) as raised: + self._engine(value) + self.assertIn("EmptyString", str(raised.exception)) + + def test_translator_takes_text(self): + translator = Translator({"Germany": "Allemagne"}, "EmptyString") + self.assertEqual("", translator.translate("Spain")) + + def test_specification_name_takes_effect_without_a_warning(self): + for text, member in SPECIFICATION_NAMES.items(): + with self.subTest(text): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + engine = self._engine_with( + missing_translation_behavior=text) + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline_with(engine))) + + def test_old_name_still_takes_effect_with_a_warning(self): + for text, member in SPECIFICATION_NAMES.items(): + with self.subTest(text): + with self.assertWarns(DeprecationWarning) as warned: + engine = self._engine_with(behavior=text) + self.assertIn( + "missing_translation_behavior", str(warned.warning)) + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline_with(engine))) + + def test_specification_name_wins_when_both_are_given(self): + with self.assertWarns(DeprecationWarning): + engine = self._engine_with( + missing_translation_behavior="EmptyString", + behavior="FlowError") + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.EMPTY_STRING], + self._missing(self._pipeline_with(engine))) + + def test_neither_name_given_is_original(self): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + engine = self._engine_with() + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.ORIGINAL], + self._missing(self._pipeline_with(engine))) + + def test_none_given_by_either_name_is_original(self): + # None means not given, so it gives Original without a warning. + translations = {"Germany": "Allemagne"} + for given in ("by position", "missing_translation_behavior", + "behavior"): + with self.subTest(given): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + if given == "by position": + engine = self._engine(None) + translator = Translator(translations, None) + else: + engine = self._engine_with(**{given: None}) + translator = Translator(translations, **{given: None}) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.ORIGINAL], + self._missing(self._pipeline_with(engine))) + self.assertEqual("Spain", translator.translate("Spain")) + + def test_translator_takes_both_names(self): + translations = {"Germany": "Allemagne"} + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + translator = Translator( + translations, missing_translation_behavior="EmptyString") + self.assertEqual("", translator.translate("Spain")) + with self.assertWarns(DeprecationWarning): + translator = Translator(translations, behavior="EmptyString") + self.assertEqual("", translator.translate("Spain")) + with self.assertWarns(DeprecationWarning): + translator = Translator( + translations, + missing_translation_behavior="Original", + behavior="EmptyString") + self.assertEqual("Spain", translator.translate("Spain")) + + def test_configuration_text_takes_effect(self): + for text, member in (("EmptyString", "EMPTY_STRING"), + ("flow_error", "FLOW_ERROR"), + ("Original", "ORIGINAL")): + with self.subTest(text): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + pipeline = PipelineBuilder().build_from_configuration( + self._configuration(text)) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior[member]], + self._missing(pipeline)) + + def test_configuration_with_an_unknown_value_is_refused(self): + with self.assertRaises(ValueError): + PipelineBuilder().build_from_configuration( + self._configuration("EmptyStrings")) + + def test_configuration_with_the_old_name_still_takes_effect(self): + with self.assertWarns(DeprecationWarning): + pipeline = PipelineBuilder().build_from_configuration( + self._configuration("EmptyString", "behavior")) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.EMPTY_STRING], + self._missing(pipeline)) + + @staticmethod + def _configuration(behavior, name="missing_translation_behavior"): + """A pipeline configuration as a configuration file gives one, so + every value in it is text, a list or a dictionary. Every option has + the specification's name, unless the behavior's name is given.""" + return { + "PipelineOptions": { + "suppress_process_exceptions": True, + "Elements": [ + { + "elementPath": __name__, + "elementName": "MockSourceElement", + "elementParameters": { + "datakey": "source", + "contents": {"Names": ["Germany", "Spain"]}, + }, + }, + { + "elementPath": ( + "fiftyone_pipeline_translation." + "translation_engine_base"), + "elementName": "TranslationEngineBase", + "elementParameters": { + "source_element_data_key": "source", + "translations": [["Names", "NamesTranslated"]], + "sources": FR_SOURCE, + "fixed_language": "fr_FR", + name: behavior, + }, + }, + ] + } + } + + if __name__ == "__main__": unittest.main()