From 575d562e3666cd7e81af88cb10031be9fd09af35 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:02:31 +0000 Subject: [PATCH 01/26] Point the documentation build at the example folders and package (#23) * Point the documentation build at the example folders and package List the cloud request engine examples and the device detection examples source folder in EXAMPLE_PATH, so Doxygen finds the files the examples name. Include each client hints example's template from its own folder, and read the location package from src/fiftyone_location. * Keep the cloud web example doc comment unbroken and name its view by folder * Keep the onpremise web example doc comment unbroken and name its view by folder --- device-detection/docs/Doxyfile | 1 + .../cloud/gettingstarted_web/app.py | 4 ++-- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/gettingstarted_web/app.py | 4 ++-- .../onpremise/useragentclienthints_web/app.py | 2 +- location/docs/Doxyfile | 2 +- pipeline/docs/Doxyfile | 1 + 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/device-detection/docs/Doxyfile b/device-detection/docs/Doxyfile index c8478bd7..b97c34ec 100644 --- a/device-detection/docs/Doxyfile +++ b/device-detection/docs/Doxyfile @@ -898,6 +898,7 @@ EXCLUDE_SYMBOLS = # command). EXAMPLE_PATH = ../fiftyone_devicedetection_examples \ + ../fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples \ ../../../documentation/src/snippets # If the value of the EXAMPLE_PATH tag contains directories, you can use the diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 340250fb..683efca1 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -42,7 +42,7 @@ # ```{py} # flowdata.evidence.add_from_dict(webevidence(request)) # ``` - +# # The module can also handling setting response headers (e.g. Accept-CH for User-Agent # Client Hints) and serving requests for client-side JavaScript and JSON resources. # ```{py} @@ -70,7 +70,7 @@ # ``` # # ## View -# @include templates/index.html +# @include cloud/gettingstarted_web/templates/index.html # # ## App diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index 448e6707..f4d8e73f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -39,7 +39,7 @@ # - [flask](https://pypi.org/project/flask/) # # ## View -# @include templates/index.html +# @include cloud/useragentclienthints_web/templates/index.html # # ## App diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index fb130b99..481f6c58 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -42,7 +42,7 @@ # ```{py} # flowdata.evidence.add_from_dict(webevidence(request)) # ``` - +# # The module can also handling setting response headers (e.g. Accept-CH for User-Agent # Client Hints) and serving requests for client-side JavaScript and JSON resources. # ```{py} @@ -70,7 +70,7 @@ # ``` # # ## View -# @include templates/index.html +# @include onpremise/gettingstarted_web/templates/index.html # # ## App diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 62f427c5..1adccf41 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -45,7 +45,7 @@ # - [flask](https://pypi.org/project/flask/) # # ## View -# @include templates/index.html +# @include onpremise/useragentclienthints_web/templates/index.html # # ## App diff --git a/location/docs/Doxyfile b/location/docs/Doxyfile index 19312f87..4d98cfcb 100644 --- a/location/docs/Doxyfile +++ b/location/docs/Doxyfile @@ -818,7 +818,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ../fiftyone_location \ +INPUT = ../src/fiftyone_location \ ../examples/cloud \ ../examples \ ../readme.md diff --git a/pipeline/docs/Doxyfile b/pipeline/docs/Doxyfile index 9ddb62fb..c62f0df0 100644 --- a/pipeline/docs/Doxyfile +++ b/pipeline/docs/Doxyfile @@ -898,6 +898,7 @@ EXCLUDE_SYMBOLS = EXAMPLE_PATH = ../fiftyone_pipeline_core/examples \ ../fiftyone_pipeline_engines_fiftyone/examples \ + ../fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples \ ../../../documentation/src/snippets # If the value of the EXAMPLE_PATH tag contains directories, you can use the From 2cd9dc8244f1ce54a313b835cd3274afc8f02212 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 3 Nov 2026 21:28:11 +0000 Subject: [PATCH 02/26] Point links to the archived repositories at where the code now lives (#25) * Point links to the archived repositories at where the code now lives The examples, readmes and example web pages linked to the archived Python repositories, and the IP Intelligence readmes linked to the archived IP Intelligence C repository. Each link now points at the same file or folder in 51degrees-python, or at the IP Intelligence readme in 51degrees-cxx, so readers land on code that is still maintained. Links to a file follow the file to its folder in this repository, for example the Device Detection examples now sit under device-detection/fiftyone_devicedetection_examples/src. Links that used the old version/4.5 branch now use main. The match metrics example closed its Markdown link with a square bracket instead of a parenthesis. The link now closes with a parenthesis. The cloud examples readme changes into the cloned folder after cloning, so its cd line now follows the new clone address. The history table in the root README still names the earlier repositories, because their tags and releases stay there. The links to the data repositories are unchanged, because those repositories are still current. * Describe the IP Intelligence performance test gap without old repository names --- .../cloud/gettingstarted_console.py | 2 +- .../cloud/gettingstarted_web/app.py | 2 +- .../cloud/metadata_console.py | 2 +- .../cloud/nativemodellookup_console.py | 2 +- .../cloud/taclookup_console.py | 2 +- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/datafileupdate_console.py | 2 +- .../onpremise/gettingstarted_console.py | 2 +- .../onpremise/gettingstarted_web/app.py | 2 +- .../onpremise/match_metrics.py | 2 +- .../onpremise/metadata_console.py | 2 +- .../onpremise/offlineprocessing.py | 2 +- .../onpremise/performance.py | 2 +- .../onpremise/useragentclienthints_web/app.py | 2 +- device-detection/readme.md | 2 +- ip-intelligence/ci/run-performance-tests.ps1 | 8 ++++---- .../fiftyone_ipintelligence_onpremise/readme.md | 2 +- ip-intelligence/readme.md | 4 ++-- location/examples/cloud/web.py | 2 +- .../examples/readme.md | 8 ++++---- .../cloud/mixed/gettingstarted_console.py | 6 +++--- .../cloud/mixed/gettingstarted_web/app.py | 2 +- .../cloud/mixed/gettingstarted_web/templates/index.html | 4 ++-- .../examples/creator_context_web/page.html | 2 +- .../examples/usagesharing/usagesharing.py | 4 ++-- pipeline/readme.md | 4 ++-- 26 files changed, 38 insertions(+), 38 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index e6bd2457..caa7c7b8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -31,7 +31,7 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/version/4.5/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 683efca1..ba86890b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -25,7 +25,7 @@ # # @include{doc} example-getting-started-web.txt # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index dd87b710..74b0df56 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -38,7 +38,7 @@ # cloud. If you are only using a single product (for example - device detection) then not all # of these keys will be relevant. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/metadata_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 2995aa9b..4d11f6f5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -29,7 +29,7 @@ # [Android devices](https://developer.android.com/reference/android/os/Build#MODEL) and # [iOS devices](https://gist.github.com/soapyigu/c99e1f45553070726f14c1bb0a54053b#file-machinename-swift) # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index cfe005b0..4563bd26 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -32,7 +32,7 @@ # 2. How to pass a TAC as evidence to the Pipeline # 3. How to retrieve device details associated with the TAC # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/version/4.5/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index f4d8e73f..730153df 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -30,7 +30,7 @@ # setting the Accept-CH header in the response. Selecting the "Make second request" # button reloads the page so those additional headers are included as evidence. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py). # # @include{doc} example-require-resourcekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 2cfdd3ed..bf9ff7a3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -148,7 +148,7 @@ # ``` # # # Location -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py). # # @include{doc} example-require-licensekey.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 57210dc7..9c695f43 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -31,7 +31,7 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/version/4.5/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index 481f6c58..c2ec0e2b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -25,7 +25,7 @@ # # @include{doc} example-getting-started-web.txt # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 1662eabb..f9f254f8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -36,7 +36,7 @@ # [https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=top] # page. # # Location -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/match_metrics.py]. +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py). from itertools import groupby from pathlib import Path diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index ea36bca1..1e5e0929 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -33,7 +33,7 @@ # keys that, when added to the evidence collection in flow data, could have some impact on the # result returned by device detection. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/metadata_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 1e639138..2487f3e9 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -42,7 +42,7 @@ # performance and predictive power using Performance Profile, Graph and Difference and Drift # settings. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index 95864e36..665cdabf 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -35,7 +35,7 @@ # and [hash dataset options](/documentation/4.5/_device_detection__hash.html#DeviceDetection_Hash_DataSetProduction_Performance) # for more information about adjusting performance. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/performance.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 1adccf41..6d10a1a7 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -36,7 +36,7 @@ # SetHeaderBrowserAccept-CH, SetHeaderPlatformAccept-CH and SetHeaderHardwareAccept-CH # properties respectively. # -# This example is available in full on [GitHub](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_examples/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py). # # @include{doc} example-require-datafile.txt # diff --git a/device-detection/readme.md b/device-detection/readme.md index 42ffce09..379b1ed1 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -13,7 +13,7 @@ The Pipeline is a generic web request intelligence and data processing solution This git repository uses submodules, to clone the git repository run: ```bash -git clone --recurse-submodules https://github.com/51Degrees/device-detection-python.git +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git ``` or if already cloned run the following to obtain all sub modules: diff --git a/ip-intelligence/ci/run-performance-tests.ps1 b/ip-intelligence/ci/run-performance-tests.ps1 index fb41eaca..c9a3dd82 100644 --- a/ip-intelligence/ci/run-performance-tests.ps1 +++ b/ip-intelligence/ci/run-performance-tests.ps1 @@ -6,7 +6,7 @@ param ( [boolean]$RunPerformance = $True ) -# Performance tests are not yet implemented for ip-intelligence-python. -# They should mirror the approach used by device-detection-python's -# performance-tests once a stable data file source is available in CI. -Write-Output "Skipping performance tests - not yet implemented for ip-intelligence-python" +# Performance tests are not yet implemented for IP Intelligence. They should +# mirror the device detection performance tests once a stable data file +# source is available in CI. +Write-Output "Skipping performance tests - not yet implemented for IP Intelligence" diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 056db37c..b7eb9ac3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -10,7 +10,7 @@ This package resolves IP addresses to location, network and registration information using a local `.ipi` data file and the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=introduction). -The native [ip-intelligence-cxx](https://github.com/51Degrees/ip-intelligence-cxx) +The native [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) library is included as a git submodule and wrapped with SWIG. ## Building diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index d01aa594..68e79e2e 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -27,14 +27,14 @@ page shows the Python versions that we currently test against. ### Git submodules The on-premise engine wraps the native -[ip-intelligence-cxx](https://github.com/51Degrees/ip-intelligence-cxx) +[51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) library, and the [ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) repository (data and evidence files used for testing) is also included, both as git submodules. When cloning, use: ``` -git clone --recurse-submodules https://github.com/51Degrees/ip-intelligence-python.git +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git ``` or if you have already cloned: diff --git a/location/examples/cloud/web.py b/location/examples/cloud/web.py index ce6bac8f..93c8e3aa 100644 --- a/location/examples/cloud/web.py +++ b/location/examples/cloud/web.py @@ -29,7 +29,7 @@ # This JavaScript is then used to obtain the additional evidence from the client and pass it back to the server. # The Pipeline running on the server then processes this evidence to populate a new JSON result object that is returned to the client. # -# This example is available in full on [GitHub](https://github.com/51Degrees/location-python/blob/main/examples/cloud/web.py) +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/location/examples/cloud/web.py) from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder from fiftyone_pipeline_core.web import webevidence diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 01071f38..cd8b5b28 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -62,8 +62,8 @@ virtual environment and install the requirements, which install the Pipeline packages from this checkout. ```sh -git clone --recurse-submodules https://github.com/51Degrees/pipeline-python.git -cd pipeline-python/fiftyone_pipeline_cloudrequestengine/examples +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git +cd 51degrees-python/pipeline/fiftyone_pipeline_cloudrequestengine/examples python -m venv .venv .venv/bin/python -m pip install -r requirements.txt ``` @@ -106,6 +106,6 @@ which can also be run by hand with `python -m tox -e py` from here. - [IP intelligence](https://51degrees.com/ip-intelligence?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=find-out-more-ip-intelligence) - [Resource keys](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=find-out-more-resource-keys) - [Pricing](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-readme.md&utm_term=find-out-more-pricing) -- Pipeline API for Python: https://github.com/51Degrees/pipeline-python -- Device detection for Python: https://github.com/51Degrees/device-detection-python +- Pipeline API for Python: https://github.com/51Degrees/51degrees-python/tree/main/pipeline +- Device detection for Python: https://github.com/51Degrees/51degrees-python/tree/main/device-detection - IP intelligence engine: https://github.com/51Degrees/51degrees-cxx diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py index 03673e18..16f0dd6b 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py @@ -33,7 +33,7 @@ # 3. How to read device and IP intelligence results from one set of flow # data # -# This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-python/blob/main/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_console.py). # # To run this example you need a resource key or a licence key. # @@ -170,9 +170,9 @@ class GettingStartedConsole(): ("Pricing", "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-examples-src-fiftyone_pipeline_examples-cloud-mixed-gettingstarted_console.py&utm_term=find-out-more-pricing"), ("Pipeline API for Python", - "https://github.com/51Degrees/pipeline-python"), + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), ("Device detection for Python", - "https://github.com/51Degrees/device-detection-python"), + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), ("IP intelligence engine", "https://github.com/51Degrees/51degrees-cxx"), ] diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py index 46a6fbfd..66856b86 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py @@ -31,7 +31,7 @@ # Each time the pipeline processes a request, one request to the cloud # service returns the results of every engine in it. # -# This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-python/blob/main/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/app.py). # # To run this example you need a resource key or a licence key. # diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html index eb93e493..88648aa2 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/src/fiftyone_pipeline_examples/cloud/mixed/gettingstarted_web/templates/index.html @@ -151,8 +151,8 @@

Find out more

  • IP intelligence
  • Resource keys
  • Pricing
    • -
  • Pipeline API for Python: https://github.com/51Degrees/pipeline-python
    • -
  • Device detection for Python: https://github.com/51Degrees/device-detection-python
    • +
  • Pipeline API for Python: https://github.com/51Degrees/51degrees-python/tree/main/pipeline
    • +
  • Device detection for Python: https://github.com/51Degrees/51degrees-python/tree/main/device-detection
  • IP intelligence engine: https://github.com/51Degrees/51degrees-cxx
    • diff --git a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html index 2baec8db..96c55f56 100644 --- a/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html +++ b/pipeline/fiftyone_pipeline_did/examples/creator_context_web/page.html @@ -148,7 +148,7 @@

    Find out more

    The source of this demo and of the code it runs on.

    diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py index fb1bee19..ade418a8 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py @@ -25,7 +25,7 @@ # @include{doc} example-usage-sharing-intro.txt # # Usage sharing is enabled by default if using some 51Degrees pipeline -# builders such as the [DeviceDetectionOnPremisePipelineBuilder](https://github.com/51Degrees/device-detection-python/blob/main/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py). +# builders such as the [DeviceDetectionOnPremisePipelineBuilder](https://github.com/51Degrees/51degrees-python/blob/main/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py). # In this example, we show how to specifically add a usage sharing # element to a Pipeline using configuration. # @@ -42,7 +42,7 @@ # For details of what each setting does, see the # [Share Usage reference](/documentation/4.5/_pipeline_api__features__usage_sharing.html) # -# This example is available in full on [GitHub](https://github.com/51Degrees/pipeline-python/blob/main/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py). +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/usagesharing.py). # # Expected output: # ``` diff --git a/pipeline/readme.md b/pipeline/readme.md index bf4c6da1..a5c12c34 100755 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -38,7 +38,7 @@ Generally, you will want to be installing one of the engines such as [device det ### From GitHub -* Clone the repository with its submodules, `git clone --recurse-submodules https://github.com/51Degrees/pipeline-python.git` +* Clone the repository with its submodules, `git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git` * Create and activate a Python virtual environment * Run `pwsh ./setup.ps1` in the root of the folder, which installs the modules from the working copy, apart from `fiftyone_pipeline_did`, together with flask @@ -56,7 +56,7 @@ The tests that call the 51Degrees cloud service read a resource key from the `re ## Examples There are several examples available that demonstrate how to make use of the Pipeline API in isolation. These are described in the table below. -If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [repository](https://github.com/51Degrees/device-detection-python) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). +If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [folder](https://github.com/51Degrees/51degrees-python/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). | Example | Description | |-----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| From 4fc6676a12293bfe79826b69796ac2735110828a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 09:16:14 +0000 Subject: [PATCH 03/26] Generate both SWIG wrappers again for 51degrees-cxx 4.5.102 (#27) * Add a temporary workflow that generates the SWIG wrappers for 51degrees-cxx 4.5.102 * Name the SWIG modules in the temporary workflow instead of reading them * Generate both SWIG wrappers again for 51degrees-cxx 4.5.102 Between 4.5.100 and 4.5.102, 51degrees-cxx removed the two EngineHash process overloads and the three EngineIpi process overloads that take required property indexes. The committed wrappers still called them, so neither on-premise package compiled against 4.5.102, which the nightly submodule update brings in #26. This moves the cxx submodule from 4110d681, the 4.5.100 release, to 1bec7b57, the 4.5.102 release, and generates hash_python_wrap.cxx, DeviceDetectionHashEngineModule.py, ipi_python_wrap.cxx and IpIntelligenceEngineModule.py again with SWIG 4.4.1 built from its v4.4.1 tag. Run against 4110d681, the same command gives files byte for byte the same as the ones committed before, so every difference here comes from cxx. Both engines call only process(evidence), which cxx keeps. * Remove the temporary SWIG wrapper workflow --- cxx | 2 +- .../DeviceDetectionHashEngineModule.py | 120 + .../hash_python_wrap.cxx | 4022 ++++++++++++++++- .../ipi_python_wrap.cxx | 345 +- 4 files changed, 3895 insertions(+), 594 deletions(-) diff --git a/cxx b/cxx index 4110d681..1bec7b57 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 4110d6813e167a803a06489318b33b83a230fa32 +Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py index 1d25c57e..d199e300 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/DeviceDetectionHashEngineModule.py @@ -1138,6 +1138,12 @@ def setUsePredictiveGraph(self, use): def setTraceRoute(self, trace): return _DeviceDetectionHashEngineModule.ConfigHashSwig_setTraceRoute(self, trace) + def setPropertyValueIndex(self, index): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_setPropertyValueIndex(self, index) + + def setIndexedProperties(self, names): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_setIndexedProperties(self, names) + def getStrings(self): return _DeviceDetectionHashEngineModule.ConfigHashSwig_getStrings(self) @@ -1173,10 +1179,121 @@ def getConcurrency(self): def getTraceRoute(self): return _DeviceDetectionHashEngineModule.ConfigHashSwig_getTraceRoute(self) + + def getPropertyValueIndex(self): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_getPropertyValueIndex(self) + + def getIndexedProperties(self): + return _DeviceDetectionHashEngineModule.ConfigHashSwig_getIndexedProperties(self) __swig_destroy__ = _DeviceDetectionHashEngineModule.delete_ConfigHashSwig # Register ConfigHashSwig in _DeviceDetectionHashEngineModule: _DeviceDetectionHashEngineModule.ConfigHashSwig_swigregister(ConfigHashSwig) +class VectorUInt32Swig(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def iterator(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_iterator(self) + def __iter__(self): + return self.iterator() + + def __nonzero__(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___nonzero__(self) + + def __bool__(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___bool__(self) + + def __len__(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___len__(self) + + def __getslice__(self, i, j): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___getslice__(self, i, j) + + def __setslice__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___setslice__(self, *args) + + def __delslice__(self, i, j): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___delslice__(self, i, j) + + def __delitem__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___delitem__(self, *args) + + def __getitem__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___getitem__(self, *args) + + def __setitem__(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig___setitem__(self, *args) + + def pop(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_pop(self) + + def append(self, x): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_append(self, x) + + def empty(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_empty(self) + + def size(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_size(self) + + def swap(self, v): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_swap(self, v) + + def begin(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_begin(self) + + def end(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_end(self) + + def rbegin(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_rbegin(self) + + def rend(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_rend(self) + + def clear(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_clear(self) + + def get_allocator(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_get_allocator(self) + + def pop_back(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_pop_back(self) + + def erase(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_erase(self, *args) + + def __init__(self, *args): + _DeviceDetectionHashEngineModule.VectorUInt32Swig_swiginit(self, _DeviceDetectionHashEngineModule.new_VectorUInt32Swig(*args)) + + def push_back(self, x): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_push_back(self, x) + + def front(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_front(self) + + def back(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_back(self) + + def assign(self, n, x): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_assign(self, n, x) + + def resize(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_resize(self, *args) + + def insert(self, *args): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_insert(self, *args) + + def reserve(self, n): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_reserve(self, n) + + def capacity(self): + return _DeviceDetectionHashEngineModule.VectorUInt32Swig_capacity(self) + __swig_destroy__ = _DeviceDetectionHashEngineModule.delete_VectorUInt32Swig + +# Register VectorUInt32Swig in _DeviceDetectionHashEngineModule: +_DeviceDetectionHashEngineModule.VectorUInt32Swig_swigregister(VectorUInt32Swig) class EngineHashSwig(EngineDeviceDetectionSwig): thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") __repr__ = _swig_repr @@ -1208,6 +1325,9 @@ def refreshDataFromMemory(self, data): def process(self, *args): return _DeviceDetectionHashEngineModule.EngineHashSwig_process(self, *args) + def getProfileIdsForPropertyValue(self, propertyName, valueName): + return _DeviceDetectionHashEngineModule.EngineHashSwig_getProfileIdsForPropertyValue(self, propertyName, valueName) + def processBase(self, evidence): return _DeviceDetectionHashEngineModule.EngineHashSwig_processBase(self, evidence) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx index 20fe7cba..e973fa1e 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/hash_python_wrap.cxx @@ -3486,19 +3486,21 @@ SWIG_Python_NonDynamicSetAttr(PyObject *obj, PyObject *name, PyObject *value) { #define SWIGTYPE_p_std__allocatorT_std__pairT_std__string_const_std__string_t_t swig_types[43] #define SWIGTYPE_p_std__allocatorT_std__string_t swig_types[44] #define SWIGTYPE_p_std__allocatorT_unsigned_char_t swig_types[45] -#define SWIGTYPE_p_std__invalid_argument swig_types[46] -#define SWIGTYPE_p_std__lessT_std__string_t swig_types[47] -#define SWIGTYPE_p_std__mapT_std__string_std__string_t swig_types[48] -#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[49] -#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[50] -#define SWIGTYPE_p_swig__SwigPyIterator swig_types[51] -#define SWIGTYPE_p_unsigned_char swig_types[52] -#define SWIGTYPE_p_unsigned_int swig_types[53] -#define SWIGTYPE_p_unsigned_long_long swig_types[54] -#define SWIGTYPE_p_unsigned_short swig_types[55] -#define SWIGTYPE_p_value_type swig_types[56] -static swig_type_info *swig_types[58]; -static swig_module_info swig_module = {swig_types, 57, 0, 0, 0, 0}; +#define SWIGTYPE_p_std__allocatorT_unsigned_int_t swig_types[46] +#define SWIGTYPE_p_std__invalid_argument swig_types[47] +#define SWIGTYPE_p_std__lessT_std__string_t swig_types[48] +#define SWIGTYPE_p_std__mapT_std__string_std__string_t swig_types[49] +#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[50] +#define SWIGTYPE_p_std__vectorT_uint32_t_t swig_types[51] +#define SWIGTYPE_p_std__vectorT_uint8_t_t swig_types[52] +#define SWIGTYPE_p_swig__SwigPyIterator swig_types[53] +#define SWIGTYPE_p_unsigned_char swig_types[54] +#define SWIGTYPE_p_unsigned_int swig_types[55] +#define SWIGTYPE_p_unsigned_long_long swig_types[56] +#define SWIGTYPE_p_unsigned_short swig_types[57] +#define SWIGTYPE_p_value_type swig_types[58] +static swig_type_info *swig_types[60]; +static swig_module_info swig_module = {swig_types, 59, 0, 0, 0, 0}; #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name) #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name) @@ -5961,6 +5963,9 @@ SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__era SWIGINTERN std::vector< std::string >::iterator std_vector_Sl_std_string_Sg__insert__SWIG_0(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::value_type const &x){ return self->insert(pos, x); } SWIGINTERN void std_vector_Sl_std_string_Sg__insert__SWIG_1(std::vector< std::string > *self,std::vector< std::string >::iterator pos,std::vector< std::string >::size_type n,std::vector< std::string >::value_type const &x){ self->insert(pos, n, x); } +#include // Use the C99 official header + + @@ -6004,9 +6009,6 @@ SWIGINTERNINLINE PyObject* } -#include // Use the C99 official header - - SWIGINTERN int SWIG_AsVal_unsigned_SS_char (PyObject * obj, unsigned char *val) { @@ -6218,6 +6220,127 @@ SWIG_AsVal_unsigned_SS_short (PyObject * obj, unsigned short *val) return res; } + +namespace swig { + template <> struct traits< unsigned int > { + typedef value_category category; + static const char* type_name() { return"unsigned int"; } + }; + template <> struct traits_asval< unsigned int > { + typedef unsigned int value_type; + static int asval(PyObject *obj, value_type *val) { + return SWIG_AsVal_unsigned_SS_int (obj, val); + } + }; + template <> struct traits_from< unsigned int > { + typedef unsigned int value_type; + static PyObject *from(const value_type& val) { + return SWIG_From_unsigned_SS_int (val); + } + }; +} + + + namespace swig { + template <> struct traits > > { + typedef pointer_category category; + static const char* type_name() { + return "std::vector<" "unsigned int" "," "std::allocator< unsigned int >" " >"; + } + }; + } + +SWIGINTERN swig::SwigPyIterator *std_vector_Sl_uint32_t_Sg__iterator(std::vector< uint32_t > *self,PyObject **PYTHON_SELF){ + return swig::make_output_iterator(self->begin(), self->begin(), self->end(), *PYTHON_SELF); + } +SWIGINTERN bool std_vector_Sl_uint32_t_Sg____nonzero__(std::vector< uint32_t > const *self){ + return !(self->empty()); + } +SWIGINTERN bool std_vector_Sl_uint32_t_Sg____bool__(std::vector< uint32_t > const *self){ + return !(self->empty()); + } +SWIGINTERN std::vector< unsigned int >::size_type std_vector_Sl_uint32_t_Sg____len__(std::vector< uint32_t > const *self){ + return self->size(); + } +SWIGINTERN std::vector< unsigned int,std::allocator< unsigned int > > *std_vector_Sl_uint32_t_Sg____getslice__(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j){ + return swig::getslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setslice____SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j){ + swig::setslice(self, i, j, 1, std::vector< unsigned int,std::allocator< unsigned int > >()); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setslice____SWIG_1(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j,std::vector< unsigned int,std::allocator< unsigned int > > const &v){ + swig::setslice(self, i, j, 1, v); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____delslice__(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::difference_type j){ + swig::delslice(self, i, j, 1); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____delitem____SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i){ + swig::erase(self, swig::getpos(self, i)); + } +SWIGINTERN std::vector< unsigned int,std::allocator< unsigned int > > *std_vector_Sl_uint32_t_Sg____getitem____SWIG_0(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return NULL; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + return swig::getslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setitem____SWIG_0(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice,std::vector< unsigned int,std::allocator< unsigned int > > const &v){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + swig::setslice(self, id, jd, step, v); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setitem____SWIG_1(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____delitem____SWIG_1(std::vector< uint32_t > *self,SWIGPY_SLICEOBJECT *slice){ + Py_ssize_t i, j, step; + if( !PySlice_Check(slice) ) { + SWIG_Error(SWIG_TypeError, "Slice object expected."); + return; + } + PySlice_GetIndices(slice, (Py_ssize_t)self->size(), &i, &j, &step); + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type id = i; + std::vector< unsigned int,std::allocator< unsigned int > >::difference_type jd = j; + swig::delslice(self, id, jd, step); + } +SWIGINTERN std::vector< unsigned int >::value_type const &std_vector_Sl_uint32_t_Sg____getitem____SWIG_1(std::vector< uint32_t > const *self,std::vector< unsigned int >::difference_type i){ + return *(swig::cgetpos(self, i)); + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg____setitem____SWIG_2(std::vector< uint32_t > *self,std::vector< unsigned int >::difference_type i,std::vector< unsigned int >::value_type const &x){ + *(swig::getpos(self,i)) = x; + } +SWIGINTERN std::vector< unsigned int >::value_type std_vector_Sl_uint32_t_Sg__pop(std::vector< uint32_t > *self){ + if (self->size() == 0) + throw std::out_of_range("pop from empty container"); + std::vector< unsigned int,std::allocator< unsigned int > >::value_type x = self->back(); + self->pop_back(); + return x; + } +SWIGINTERN void std_vector_Sl_uint32_t_Sg__append(std::vector< uint32_t > *self,std::vector< unsigned int >::value_type const &x){ + self->push_back(x); + } +SWIGINTERN std::vector< unsigned int >::iterator std_vector_Sl_uint32_t_Sg__erase__SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator pos){ return self->erase(pos); } +SWIGINTERN std::vector< unsigned int >::iterator std_vector_Sl_uint32_t_Sg__erase__SWIG_1(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator first,std::vector< unsigned int >::iterator last){ return self->erase(first, last); } +SWIGINTERN std::vector< unsigned int >::iterator std_vector_Sl_uint32_t_Sg__insert__SWIG_0(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator pos,std::vector< unsigned int >::value_type const &x){ return self->insert(pos, x); } +SWIGINTERN void std_vector_Sl_uint32_t_Sg__insert__SWIG_1(std::vector< uint32_t > *self,std::vector< unsigned int >::iterator pos,std::vector< unsigned int >::size_type n,std::vector< unsigned int >::value_type const &x){ self->insert(pos, n, x); } #ifdef __cplusplus extern "C" { #endif @@ -25527,6 +25650,130 @@ SWIGINTERN PyObject *_wrap_ConfigHashSwig_setTraceRoute(PyObject *self, PyObject } +SWIGINTERN PyObject *_wrap_ConfigHashSwig_setPropertyValueIndex(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ConfigHash *arg1 = 0 ; + bool arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ConfigHashSwig_setPropertyValueIndex", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_setPropertyValueIndex" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); + ecode2 = SWIG_AsVal_bool(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ConfigHashSwig_setPropertyValueIndex" "', argument " "2"" of type '" "bool""'"); + } + arg2 = static_cast< bool >(val2); + { + try { + (arg1)->setPropertyValueIndex(arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ConfigHashSwig_setIndexedProperties(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ConfigHash *arg1 = 0 ; + std::vector< std::string,std::allocator< std::string > > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ConfigHashSwig_setIndexedProperties", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_setIndexedProperties" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); + { + std::vector< std::string,std::allocator< std::string > > *ptr = (std::vector< std::string,std::allocator< std::string > > *)0; + int res = swig::asptr(swig_obj[1], &ptr); + if (!SWIG_IsOK(res) || !ptr) { + SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "ConfigHashSwig_setIndexedProperties" "', argument " "2"" of type '" "std::vector< std::string,std::allocator< std::string > >""'"); + } + arg2 = *ptr; + if (SWIG_IsNewObj(res)) delete ptr; + } + { + try { + (arg1)->setIndexedProperties(SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + SWIGINTERN PyObject *_wrap_ConfigHashSwig_getStrings(PyObject *self, PyObject *args) { PyObject *resultobj = 0; ConfigHash *arg1 = 0 ; @@ -26187,24 +26434,25 @@ SWIGINTERN PyObject *_wrap_ConfigHashSwig_getTraceRoute(PyObject *self, PyObject } -SWIGINTERN PyObject *_wrap_delete_ConfigHashSwig(PyObject *self, PyObject *args) { +SWIGINTERN PyObject *_wrap_ConfigHashSwig_getPropertyValueIndex(PyObject *self, PyObject *args) { PyObject *resultobj = 0; ConfigHash *arg1 = 0 ; void *argp1 = 0 ; int res1 = 0 ; PyObject *swig_obj[1] ; + bool result; (void)self; if (!args) SWIG_fail; swig_obj[0] = args; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, SWIG_POINTER_DISOWN | 0 ); + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ConfigHashSwig" "', argument " "1"" of type '" "ConfigHash *""'"); + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_getPropertyValueIndex" "', argument " "1"" of type '" "ConfigHash *""'"); } arg1 = reinterpret_cast< ConfigHash * >(argp1); { try { - delete arg1;; + result = (bool)(arg1)->getPropertyValueIndex();; } catch(FatalException& e) { SWIG_exception(SWIG_RuntimeError, e.what()); @@ -26234,53 +26482,3401 @@ SWIGINTERN PyObject *_wrap_delete_ConfigHashSwig(PyObject *self, PyObject *args) SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_Py_Void(); + resultobj = SWIG_From_bool(static_cast< bool >(result)); return resultobj; fail: return NULL; } -SWIGINTERN PyObject *ConfigHashSwig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - PyObject *obj = NULL; - if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; - SWIG_TypeNewClientData(SWIGTYPE_p_ConfigHash, SWIG_NewClientData(obj)); - return SWIG_Py_Void(); -} - -SWIGINTERN PyObject *ConfigHashSwig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { - return SWIG_Python_InitShadowInstance(args); -} - -SWIGINTERN PyObject *_wrap_new_EngineHashSwig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { +SWIGINTERN PyObject *_wrap_ConfigHashSwig_getIndexedProperties(PyObject *self, PyObject *args) { PyObject *resultobj = 0; - std::string *arg1 = 0 ; - ConfigHash *arg2 = 0 ; - RequiredPropertiesConfig *arg3 = 0 ; - int res1 = SWIG_OLDOBJ ; - void *argp2 = 0 ; - int res2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - EngineHash *result = 0 ; + ConfigHash *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< std::string,std::allocator< std::string > > result; (void)self; - if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ConfigHashSwig_getIndexedProperties" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); { - std::string *ptr = (std::string *)0; - res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + try { + result = (arg1)->getIndexedProperties();; } - if (!ptr) { - SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); } - arg1 = ptr; - } - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_ConfigHash, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "new_EngineHashSwig" "', argument " "2"" of type '" "ConfigHash *""'"); - } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = swig::from(static_cast< std::vector< std::string,std::allocator< std::string > > >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ConfigHashSwig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + ConfigHash *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_ConfigHash, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ConfigHashSwig" "', argument " "1"" of type '" "ConfigHash *""'"); + } + arg1 = reinterpret_cast< ConfigHash * >(argp1); + { + try { + delete arg1;; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ConfigHashSwig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_ConfigHash, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ConfigHashSwig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_iterator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + PyObject **arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + swig::SwigPyIterator *result = 0 ; + + arg2 = &swig_obj[0]; + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_iterator" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (swig::SwigPyIterator *)std_vector_Sl_uint32_t_Sg__iterator(arg1,arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + if (!result) SWIG_fail; + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_swig__SwigPyIterator, SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___nonzero__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___nonzero__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (bool)std_vector_Sl_uint32_t_Sg____nonzero__((std::vector< unsigned int > const *)arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___bool__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___bool__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (bool)std_vector_Sl_uint32_t_Sg____bool__((std::vector< unsigned int > const *)arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___len__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___len__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = std_vector_Sl_uint32_t_Sg____len__((std::vector< unsigned int > const *)arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + std::vector< unsigned int,std::allocator< unsigned int > > *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___getslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___getslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___getslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___getslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + try { + try { + result = (std::vector< unsigned int,std::allocator< unsigned int > > *)std_vector_Sl_uint32_t_Sg____getslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setslice____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___setslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___setslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + try { + try { + std_vector_Sl_uint32_t_Sg____setslice____SWIG_0(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setslice____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + std::vector< unsigned int,std::allocator< unsigned int > > *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + int res4 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___setslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___setslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + std::vector< unsigned int,std::allocator< unsigned int > > *ptr = (std::vector< unsigned int,std::allocator< unsigned int > > *)0; + res4 = swig::asptr(swig_obj[3], &ptr); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "VectorUInt32Swig___setslice__" "', argument " "4"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "VectorUInt32Swig___setslice__" "', argument " "4"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + arg4 = ptr; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____setslice____SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(std::vector< unsigned int,std::allocator< unsigned int > > const &)*arg4); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res4)) delete arg4; + return resultobj; +fail: + if (SWIG_IsNewObj(res4)) delete arg4; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setslice__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___setslice__", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___setslice____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + int res = swig::asptr(argv[3], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_VectorUInt32Swig___setslice____SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___setslice__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__setslice__(std::vector< unsigned int >::difference_type,std::vector< unsigned int >::difference_type)\n" + " std::vector< uint32_t >::__setslice__(std::vector< unsigned int >::difference_type,std::vector< unsigned int >::difference_type,std::vector< unsigned int,std::allocator< unsigned int > > const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delslice__(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::difference_type arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + ptrdiff_t val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___delslice__", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___delslice__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___delslice__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_ptrdiff_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___delslice__" "', argument " "3"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::difference_type >(val3); + { + try { + try { + std_vector_Sl_uint32_t_Sg____delslice__(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___delitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___delitem__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + { + try { + try { + std_vector_Sl_uint32_t_Sg____delitem____SWIG_0(arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned int,std::allocator< unsigned int > > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___getitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___getitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + try { + try { + result = (std::vector< unsigned int,std::allocator< unsigned int > > *)std_vector_Sl_uint32_t_Sg____getitem____SWIG_0(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem____SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + std::vector< unsigned int,std::allocator< unsigned int > > *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + int res3 = SWIG_OLDOBJ ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + std::vector< unsigned int,std::allocator< unsigned int > > *ptr = (std::vector< unsigned int,std::allocator< unsigned int > > *)0; + res3 = swig::asptr(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "VectorUInt32Swig___setitem__" "', argument " "3"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "VectorUInt32Swig___setitem__" "', argument " "3"" of type '" "std::vector< unsigned int,std::allocator< unsigned int > > const &""'"); + } + arg3 = ptr; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____setitem____SWIG_0(arg1,arg2,(std::vector< unsigned int,std::allocator< unsigned int > > const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + if (SWIG_IsNewObj(res3)) delete arg3; + return resultobj; +fail: + if (SWIG_IsNewObj(res3)) delete arg3; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___setitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____setitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SWIGPY_SLICEOBJECT *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___delitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + if (!PySlice_Check(swig_obj[1])) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig___delitem__" "', argument " "2"" of type '" "SWIGPY_SLICEOBJECT *""'"); + } + arg2 = (SWIGPY_SLICEOBJECT *) swig_obj[1]; + } + { + try { + try { + std_vector_Sl_uint32_t_Sg____delitem____SWIG_1(arg1,arg2); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + } catch(std::invalid_argument &_e) { + SWIG_exception_fail(SWIG_ValueError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___delitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___delitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_VectorUInt32Swig___delitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___delitem____SWIG_0(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___delitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__delitem__(std::vector< unsigned int >::difference_type)\n" + " std::vector< uint32_t >::__delitem__(SWIGPY_SLICEOBJECT *)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getitem____SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___getitem__" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___getitem__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + { + try { + try { + result = (std::vector< unsigned int >::value_type *) &std_vector_Sl_uint32_t_Sg____getitem____SWIG_1((std::vector< unsigned int > const *)arg1,SWIG_STD_MOVE(arg2)); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(*result)); + if (!swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]) && PyErr_Occurred()) { + SWIG_fail; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___getitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___getitem__", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_VectorUInt32Swig___getitem____SWIG_0(self, argc, argv); + } + } + } + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___getitem____SWIG_1(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___getitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__getitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< uint32_t >::__getitem__(std::vector< unsigned int >::difference_type) const\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem____SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::difference_type arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + ptrdiff_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig___setitem__" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_ptrdiff_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig___setitem__" "', argument " "2"" of type '" "std::vector< unsigned int >::difference_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::difference_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig___setitem__" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + try { + std_vector_Sl_uint32_t_Sg____setitem____SWIG_2(arg1,SWIG_STD_MOVE(arg2),(unsigned int const &)*arg3); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig___setitem__(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig___setitem__", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + return _wrap_VectorUInt32Swig___setitem____SWIG_1(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + _v = PySlice_Check(argv[1]); + } + if (_v) { + int res = swig::asptr(argv[2], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_VectorUInt32Swig___setitem____SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_ptrdiff_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig___setitem____SWIG_2(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig___setitem__'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::__setitem__(SWIGPY_SLICEOBJECT *,std::vector< unsigned int,std::allocator< unsigned int > > const &)\n" + " std::vector< uint32_t >::__setitem__(SWIGPY_SLICEOBJECT *)\n" + " std::vector< uint32_t >::__setitem__(std::vector< unsigned int >::difference_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_pop(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::value_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_pop" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + try { + result = (std::vector< unsigned int >::value_type)std_vector_Sl_uint32_t_Sg__pop(arg1); + } catch(std::out_of_range &_e) { + SWIG_exception_fail(SWIG_IndexError, (&_e)->what()); + }; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_append(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned int >::value_type temp2 ; + unsigned int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_append", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_append" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_append" "', argument " "2"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned int >::value_type >(val2); + arg2 = &temp2; + { + try { + std_vector_Sl_uint32_t_Sg__append(arg1,(unsigned int const &)*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **SWIGUNUSEDPARM(swig_obj)) { + PyObject *resultobj = 0; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 0) || (nobjs > 0)) SWIG_fail; + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + int res1 = SWIG_OLDOBJ ; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + { + std::vector< unsigned int,std::allocator< unsigned int > > *ptr = (std::vector< unsigned int,std::allocator< unsigned int > > *)0; + res1 = swig::asptr(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< uint32_t > const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< uint32_t > const &""'"); + } + arg1 = ptr; + } + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >((std::vector< uint32_t > const &)*arg1);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + if (SWIG_IsNewObj(res1)) delete arg1; + return resultobj; +fail: + if (SWIG_IsNewObj(res1)) delete arg1; + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_empty(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + bool result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_empty" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (bool)((std::vector< uint32_t > const *)arg1)->empty();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_size(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_size" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = ((std::vector< uint32_t > const *)arg1)->size();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_swap(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< uint32_t > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + void *argp2 = 0 ; + int res2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_swap", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_swap" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2, SWIGTYPE_p_std__vectorT_uint32_t_t, 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "VectorUInt32Swig_swap" "', argument " "2"" of type '" "std::vector< uint32_t > &""'"); + } + if (!argp2) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "VectorUInt32Swig_swap" "', argument " "2"" of type '" "std::vector< uint32_t > &""'"); + } + arg2 = reinterpret_cast< std::vector< uint32_t > * >(argp2); + { + try { + (arg1)->swap(*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_begin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_begin" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->begin();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_end(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_end" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->end();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_rbegin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_rbegin" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->rbegin();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_rend(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::vector< unsigned int >::reverse_iterator > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_rend" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (arg1)->rend();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::reverse_iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_clear(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_clear" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + (arg1)->clear();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_get_allocator(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + SwigValueWrapper< std::allocator< unsigned int > > result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_get_allocator" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = ((std::vector< uint32_t > const *)arg1)->get_allocator();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj((new std::vector< unsigned int >::allocator_type(result)), SWIGTYPE_p_std__allocatorT_unsigned_int_t, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< unsigned int >::size_type arg1 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 1) || (nobjs > 1)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg1 = static_cast< std::vector< unsigned int >::size_type >(val1); + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >(SWIG_STD_MOVE(arg1));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_pop_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_pop_back" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + (arg1)->pop_back();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_resize__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_resize" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_resize" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + { + try { + (arg1)->resize(SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_erase__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_erase" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + { + try { + result = std_vector_Sl_uint32_t_Sg__erase__SWIG_0(arg1,SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_erase__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + swig::SwigPyIterator *iter3 = 0 ; + int res3 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_erase" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + res3 = SWIG_ConvertPtr(swig_obj[2], SWIG_as_voidptrptr(&iter3), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res3) || !iter3) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "3"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter3); + if (iter_t) { + arg3 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_erase" "', argument " "3"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + { + try { + result = std_vector_Sl_uint32_t_Sg__erase__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_erase(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_erase", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_VectorUInt32Swig_erase__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[2], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + return _wrap_VectorUInt32Swig_erase__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig_erase'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::erase(std::vector< unsigned int >::iterator)\n" + " std::vector< uint32_t >::erase(std::vector< unsigned int >::iterator,std::vector< unsigned int >::iterator)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< unsigned int >::size_type arg1 ; + std::vector< unsigned int >::value_type *arg2 = 0 ; + size_t val1 ; + int ecode1 = 0 ; + std::vector< unsigned int >::value_type temp2 ; + unsigned int val2 ; + int ecode2 = 0 ; + std::vector< uint32_t > *result = 0 ; + + (void)self; + if ((nobjs < 2) || (nobjs > 2)) SWIG_fail; + ecode1 = SWIG_AsVal_size_t(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "new_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg1 = static_cast< std::vector< unsigned int >::size_type >(val1); + ecode2 = SWIG_AsVal_unsigned_SS_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "new_VectorUInt32Swig" "', argument " "2"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned int >::value_type >(val2); + arg2 = &temp2; + { + try { + result = (std::vector< uint32_t > *)new std::vector< uint32_t >(SWIG_STD_MOVE(arg1),(std::vector< unsigned int >::value_type const &)*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_new_VectorUInt32Swig(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "new_VectorUInt32Swig", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 0) { + return _wrap_new_VectorUInt32Swig__SWIG_0(self, argc, argv); + } + if (argc == 1) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_VectorUInt32Swig__SWIG_2(self, argc, argv); + } + } + if (argc == 1) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_new_VectorUInt32Swig__SWIG_1(self, argc, argv); + } + } + if (argc == 2) { + int _v = 0; + { + int res = SWIG_AsVal_size_t(argv[0], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_new_VectorUInt32Swig__SWIG_3(self, argc, argv); + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'new_VectorUInt32Swig'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::vector()\n" + " std::vector< uint32_t >::vector(std::vector< uint32_t > const &)\n" + " std::vector< uint32_t >::vector(std::vector< unsigned int >::size_type)\n" + " std::vector< uint32_t >::vector(std::vector< unsigned int >::size_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_push_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::value_type *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + std::vector< unsigned int >::value_type temp2 ; + unsigned int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_push_back", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_push_back" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_unsigned_SS_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_push_back" "', argument " "2"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp2 = static_cast< std::vector< unsigned int >::value_type >(val2); + arg2 = &temp2; + { + try { + (arg1)->push_back((std::vector< unsigned int >::value_type const &)*arg2);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_front(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_front" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (std::vector< unsigned int >::value_type *) &((std::vector< uint32_t > const *)arg1)->front();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(*result)); + if (!swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]) && PyErr_Occurred()) { + SWIG_fail; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_back(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::value_type *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_back" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = (std::vector< unsigned int >::value_type *) &((std::vector< uint32_t > const *)arg1)->back();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(*result)); + if (!swig::container_owner::value_type>::category>::back_reference(resultobj, swig_obj[0]) && PyErr_Occurred()) { + SWIG_fail; + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_assign(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_assign", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_assign" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_assign" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_assign" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + (arg1)->assign(SWIG_STD_MOVE(arg2),(std::vector< unsigned int >::value_type const &)*arg3);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_resize__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_resize" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_resize" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_resize" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + (arg1)->resize(SWIG_STD_MOVE(arg2),(std::vector< unsigned int >::value_type const &)*arg3);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_resize(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[4] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_resize", 0, 3, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_resize__SWIG_0(self, argc, argv); + } + } + } + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[1], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_resize__SWIG_1(self, argc, argv); + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig_resize'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::resize(std::vector< unsigned int >::size_type)\n" + " std::vector< uint32_t >::resize(std::vector< unsigned int >::size_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_insert__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + std::vector< unsigned int >::value_type *arg3 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + std::vector< unsigned int >::value_type temp3 ; + unsigned int val3 ; + int ecode3 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_insert" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_unsigned_SS_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_insert" "', argument " "3"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp3 = static_cast< std::vector< unsigned int >::value_type >(val3); + arg3 = &temp3; + { + try { + result = std_vector_Sl_uint32_t_Sg__insert__SWIG_0(arg1,SWIG_STD_MOVE(arg2),(unsigned int const &)*arg3);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_NewPointerObj(swig::make_output_iterator(static_cast< const std::vector< unsigned int >::iterator & >(result)), + swig::SwigPyIterator::descriptor(),SWIG_POINTER_OWN); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_insert__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + SwigValueWrapper< std::vector< unsigned int >::iterator > arg2 ; + std::vector< unsigned int >::size_type arg3 ; + std::vector< unsigned int >::value_type *arg4 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + swig::SwigPyIterator *iter2 = 0 ; + int res2 ; + size_t val3 ; + int ecode3 = 0 ; + std::vector< unsigned int >::value_type temp4 ; + unsigned int val4 ; + int ecode4 = 0 ; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_insert" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + res2 = SWIG_ConvertPtr(swig_obj[1], SWIG_as_voidptrptr(&iter2), swig::SwigPyIterator::descriptor(), 0); + if (!SWIG_IsOK(res2) || !iter2) { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } else { + swig::SwigPyIterator_T::iterator > *iter_t = dynamic_cast::iterator > *>(iter2); + if (iter_t) { + arg2 = iter_t->get_current(); + } else { + SWIG_exception_fail(SWIG_ArgError(SWIG_TypeError), "in method '" "VectorUInt32Swig_insert" "', argument " "2"" of type '" "std::vector< unsigned int >::iterator""'"); + } + } + ecode3 = SWIG_AsVal_size_t(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "VectorUInt32Swig_insert" "', argument " "3"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg3 = static_cast< std::vector< unsigned int >::size_type >(val3); + ecode4 = SWIG_AsVal_unsigned_SS_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "VectorUInt32Swig_insert" "', argument " "4"" of type '" "std::vector< unsigned int >::value_type""'"); + } + temp4 = static_cast< std::vector< unsigned int >::value_type >(val4); + arg4 = &temp4; + { + try { + std_vector_Sl_uint32_t_Sg__insert__SWIG_1(arg1,SWIG_STD_MOVE(arg2),SWIG_STD_MOVE(arg3),(unsigned int const &)*arg4);; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_insert(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_insert", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_insert__SWIG_0(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + int res = swig::asptr(argv[0], (std::vector< unsigned int,std::allocator< unsigned int > >**)(0)); + _v = SWIG_CheckState(res); + if (_v) { + swig::SwigPyIterator *iter = 0; + int res = SWIG_ConvertPtr(argv[1], SWIG_as_voidptrptr(&iter), swig::SwigPyIterator::descriptor(), 0); + _v = (SWIG_IsOK(res) && iter && (dynamic_cast::iterator > *>(iter) != 0)); + if (_v) { + { + int res = SWIG_AsVal_size_t(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + { + int res = SWIG_AsVal_unsigned_SS_int(argv[3], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_VectorUInt32Swig_insert__SWIG_1(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'VectorUInt32Swig_insert'.\n" + " Possible C/C++ prototypes are:\n" + " std::vector< uint32_t >::insert(std::vector< unsigned int >::iterator,std::vector< unsigned int >::value_type const &)\n" + " std::vector< uint32_t >::insert(std::vector< unsigned int >::iterator,std::vector< unsigned int >::size_type,std::vector< unsigned int >::value_type const &)\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_reserve(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + std::vector< unsigned int >::size_type arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + size_t val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "VectorUInt32Swig_reserve", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_reserve" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + ecode2 = SWIG_AsVal_size_t(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "VectorUInt32Swig_reserve" "', argument " "2"" of type '" "std::vector< unsigned int >::size_type""'"); + } + arg2 = static_cast< std::vector< unsigned int >::size_type >(val2); + { + try { + (arg1)->reserve(SWIG_STD_MOVE(arg2));; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_VectorUInt32Swig_capacity(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::vector< unsigned int >::size_type result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "VectorUInt32Swig_capacity" "', argument " "1"" of type '" "std::vector< uint32_t > const *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + result = ((std::vector< uint32_t > const *)arg1)->capacity();; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_VectorUInt32Swig(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::vector< uint32_t > *arg1 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_VectorUInt32Swig" "', argument " "1"" of type '" "std::vector< uint32_t > *""'"); + } + arg1 = reinterpret_cast< std::vector< uint32_t > * >(argp1); + { + try { + delete arg1;; + } + catch(FatalException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NotImplementedException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(InvalidPropertyException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(EvidenceException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(TooManyValuesException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(NoValuesAvailableException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(StatusCodeException& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(runtime_error& e) { + SWIG_exception(SWIG_RuntimeError, e.what()); + } + catch(invalid_argument& e) { + SWIG_exception(SWIG_ValueError, e.what()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *VectorUInt32Swig_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_std__vectorT_uint32_t_t, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *VectorUInt32Swig_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_EngineHashSwig__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + std::string *arg1 = 0 ; + ConfigHash *arg2 = 0 ; + RequiredPropertiesConfig *arg3 = 0 ; + int res1 = SWIG_OLDOBJ ; + void *argp2 = 0 ; + int res2 = 0 ; + void *argp3 = 0 ; + int res3 = 0 ; + EngineHash *result = 0 ; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + { + std::string *ptr = (std::string *)0; + res1 = SWIG_AsPtr_std_string(swig_obj[0], &ptr); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "new_EngineHashSwig" "', argument " "1"" of type '" "std::string const &""'"); + } + arg1 = ptr; + } + res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_ConfigHash, 0 | 0 ); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "new_EngineHashSwig" "', argument " "2"" of type '" "ConfigHash *""'"); + } arg2 = reinterpret_cast< ConfigHash * >(argp2); res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_RequiredPropertiesConfig, 0 | 0 ); if (!SWIG_IsOK(res3)) { @@ -27030,125 +30626,95 @@ SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_1(PyObject *self, Py_ssi } -SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_2(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineHash *arg1 = 0 ; - EvidenceDeviceDetection *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; - void *argp1 = 0 ; - int res1 = 0 ; - void *argp2 = 0 ; - int res2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsHash *result = 0 ; +SWIGINTERN PyObject *_wrap_EngineHashSwig_process(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[3] = { + 0 + }; - (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineHash, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineHashSwig_process" "', argument " "1"" of type '" "EngineHash *""'"); - } - arg1 = reinterpret_cast< EngineHash * >(argp1); - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_EvidenceDeviceDetection, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineHashSwig_process" "', argument " "2"" of type '" "EvidenceDeviceDetection *""'"); - } - arg2 = reinterpret_cast< EvidenceDeviceDetection * >(argp2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineHashSwig_process" "', argument " "3"" of type '" "int const *""'"); - } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineHashSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); - { - try { - result = (ResultsHash *)(arg1)->process(arg2,(int const *)arg3,arg4);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); + if (!(argc = SWIG_Python_UnpackTuple(args, "EngineHashSwig_process", 0, 2, argv))) SWIG_fail; + --argc; + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); + _v = SWIG_CheckState(res); + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceDeviceDetection, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_EngineHashSwig_process__SWIG_0(self, argc, argv); + } } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); + } + if (argc == 2) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_EngineHashSwig_process__SWIG_1(self, argc, argv); + } } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsHash, SWIG_POINTER_OWN | 0 ); - return resultobj; + fail: - return NULL; + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'EngineHashSwig_process'.\n" + " Possible C/C++ prototypes are:\n" + " EngineHash::process(EvidenceDeviceDetection *)\n" + " EngineHash::process(char const *)\n"); + return 0; } -SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { +SWIGINTERN PyObject *_wrap_EngineHashSwig_getProfileIdsForPropertyValue(PyObject *self, PyObject *args) { PyObject *resultobj = 0; EngineHash *arg1 = 0 ; - char *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; + std::string *arg2 = 0 ; + std::string *arg3 = 0 ; void *argp1 = 0 ; int res1 = 0 ; - int res2 ; - char *buf2 = 0 ; - int alloc2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsHash *result = 0 ; + int res2 = SWIG_OLDOBJ ; + int res3 = SWIG_OLDOBJ ; + PyObject *swig_obj[3] ; + std::vector< uint32_t,std::allocator< uint32_t > > result; (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + if (!SWIG_Python_UnpackTuple(args, "EngineHashSwig_getProfileIdsForPropertyValue", 3, 3, swig_obj)) SWIG_fail; res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineHash, 0 | 0 ); if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineHashSwig_process" "', argument " "1"" of type '" "EngineHash *""'"); + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "1"" of type '" "EngineHash *""'"); } arg1 = reinterpret_cast< EngineHash * >(argp1); - res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineHashSwig_process" "', argument " "2"" of type '" "char const *""'"); + { + std::string *ptr = (std::string *)0; + res2 = SWIG_AsPtr_std_string(swig_obj[1], &ptr); + if (!SWIG_IsOK(res2)) { + SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "2"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "2"" of type '" "std::string const &""'"); + } + arg2 = ptr; } - arg2 = reinterpret_cast< char * >(buf2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineHashSwig_process" "', argument " "3"" of type '" "int const *""'"); + { + std::string *ptr = (std::string *)0; + res3 = SWIG_AsPtr_std_string(swig_obj[2], &ptr); + if (!SWIG_IsOK(res3)) { + SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "3"" of type '" "std::string const &""'"); + } + if (!ptr) { + SWIG_exception_fail(SWIG_NullReferenceError, "invalid null reference " "in method '" "EngineHashSwig_getProfileIdsForPropertyValue" "', argument " "3"" of type '" "std::string const &""'"); + } + arg3 = ptr; } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineHashSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); { try { - result = (ResultsHash *)(arg1)->process((char const *)arg2,(int const *)arg3,arg4);; + result = (arg1)->getProfileIdsForPropertyValue((std::string const &)*arg2,(std::string const &)*arg3);; } catch(FatalException& e) { SWIG_exception(SWIG_RuntimeError, e.what()); @@ -27178,111 +30744,17 @@ SWIGINTERN PyObject *_wrap_EngineHashSwig_process__SWIG_3(PyObject *self, Py_ssi SWIG_exception(SWIG_ValueError, e.what()); } } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsHash, SWIG_POINTER_OWN | 0 ); - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + resultobj = swig::from(static_cast< std::vector< unsigned int,std::allocator< unsigned int > > >(result)); + if (SWIG_IsNewObj(res2)) delete arg2; + if (SWIG_IsNewObj(res3)) delete arg3; return resultobj; fail: - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; + if (SWIG_IsNewObj(res2)) delete arg2; + if (SWIG_IsNewObj(res3)) delete arg3; return NULL; } -SWIGINTERN PyObject *_wrap_EngineHashSwig_process(PyObject *self, PyObject *args) { - Py_ssize_t argc; - PyObject *argv[5] = { - 0 - }; - - if (!(argc = SWIG_Python_UnpackTuple(args, "EngineHashSwig_process", 0, 4, argv))) SWIG_fail; - --argc; - if (argc == 2) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceDeviceDetection, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_0(self, argc, argv); - } - } - } - if (argc == 2) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_1(self, argc, argv); - } - } - } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceDeviceDetection, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_2(self, argc, argv); - } - } - } - } - } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineHash, 0); - _v = SWIG_CheckState(res); - if (_v) { - int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineHashSwig_process__SWIG_3(self, argc, argv); - } - } - } - } - } - -fail: - SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'EngineHashSwig_process'.\n" - " Possible C/C++ prototypes are:\n" - " EngineHash::process(EvidenceDeviceDetection *)\n" - " EngineHash::process(char const *)\n" - " EngineHash::process(EvidenceDeviceDetection *,int const *,int)\n" - " EngineHash::process(char const *,int const *,int)\n"); - return 0; -} - - SWIGINTERN PyObject *_wrap_EngineHashSwig_processBase(PyObject *self, PyObject *args) { PyObject *resultobj = 0; EngineHash *arg1 = 0 ; @@ -28577,6 +32049,8 @@ static PyMethodDef SwigMethods[] = { { "ConfigHashSwig_setUsePerformanceGraph", _wrap_ConfigHashSwig_setUsePerformanceGraph, METH_VARARGS, NULL}, { "ConfigHashSwig_setUsePredictiveGraph", _wrap_ConfigHashSwig_setUsePredictiveGraph, METH_VARARGS, NULL}, { "ConfigHashSwig_setTraceRoute", _wrap_ConfigHashSwig_setTraceRoute, METH_VARARGS, NULL}, + { "ConfigHashSwig_setPropertyValueIndex", _wrap_ConfigHashSwig_setPropertyValueIndex, METH_VARARGS, NULL}, + { "ConfigHashSwig_setIndexedProperties", _wrap_ConfigHashSwig_setIndexedProperties, METH_VARARGS, NULL}, { "ConfigHashSwig_getStrings", _wrap_ConfigHashSwig_getStrings, METH_O, NULL}, { "ConfigHashSwig_getProperties", _wrap_ConfigHashSwig_getProperties, METH_O, NULL}, { "ConfigHashSwig_getValues", _wrap_ConfigHashSwig_getValues, METH_O, NULL}, @@ -28589,9 +32063,46 @@ static PyMethodDef SwigMethods[] = { { "ConfigHashSwig_getUsePredictiveGraph", _wrap_ConfigHashSwig_getUsePredictiveGraph, METH_O, NULL}, { "ConfigHashSwig_getConcurrency", _wrap_ConfigHashSwig_getConcurrency, METH_O, NULL}, { "ConfigHashSwig_getTraceRoute", _wrap_ConfigHashSwig_getTraceRoute, METH_O, NULL}, + { "ConfigHashSwig_getPropertyValueIndex", _wrap_ConfigHashSwig_getPropertyValueIndex, METH_O, NULL}, + { "ConfigHashSwig_getIndexedProperties", _wrap_ConfigHashSwig_getIndexedProperties, METH_O, NULL}, { "delete_ConfigHashSwig", _wrap_delete_ConfigHashSwig, METH_O, NULL}, { "ConfigHashSwig_swigregister", ConfigHashSwig_swigregister, METH_O, NULL}, { "ConfigHashSwig_swiginit", ConfigHashSwig_swiginit, METH_VARARGS, NULL}, + { "VectorUInt32Swig_iterator", _wrap_VectorUInt32Swig_iterator, METH_O, NULL}, + { "VectorUInt32Swig___nonzero__", _wrap_VectorUInt32Swig___nonzero__, METH_O, NULL}, + { "VectorUInt32Swig___bool__", _wrap_VectorUInt32Swig___bool__, METH_O, NULL}, + { "VectorUInt32Swig___len__", _wrap_VectorUInt32Swig___len__, METH_O, NULL}, + { "VectorUInt32Swig___getslice__", _wrap_VectorUInt32Swig___getslice__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___setslice__", _wrap_VectorUInt32Swig___setslice__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___delslice__", _wrap_VectorUInt32Swig___delslice__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___delitem__", _wrap_VectorUInt32Swig___delitem__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___getitem__", _wrap_VectorUInt32Swig___getitem__, METH_VARARGS, NULL}, + { "VectorUInt32Swig___setitem__", _wrap_VectorUInt32Swig___setitem__, METH_VARARGS, NULL}, + { "VectorUInt32Swig_pop", _wrap_VectorUInt32Swig_pop, METH_O, NULL}, + { "VectorUInt32Swig_append", _wrap_VectorUInt32Swig_append, METH_VARARGS, NULL}, + { "VectorUInt32Swig_empty", _wrap_VectorUInt32Swig_empty, METH_O, NULL}, + { "VectorUInt32Swig_size", _wrap_VectorUInt32Swig_size, METH_O, NULL}, + { "VectorUInt32Swig_swap", _wrap_VectorUInt32Swig_swap, METH_VARARGS, NULL}, + { "VectorUInt32Swig_begin", _wrap_VectorUInt32Swig_begin, METH_O, NULL}, + { "VectorUInt32Swig_end", _wrap_VectorUInt32Swig_end, METH_O, NULL}, + { "VectorUInt32Swig_rbegin", _wrap_VectorUInt32Swig_rbegin, METH_O, NULL}, + { "VectorUInt32Swig_rend", _wrap_VectorUInt32Swig_rend, METH_O, NULL}, + { "VectorUInt32Swig_clear", _wrap_VectorUInt32Swig_clear, METH_O, NULL}, + { "VectorUInt32Swig_get_allocator", _wrap_VectorUInt32Swig_get_allocator, METH_O, NULL}, + { "VectorUInt32Swig_pop_back", _wrap_VectorUInt32Swig_pop_back, METH_O, NULL}, + { "VectorUInt32Swig_erase", _wrap_VectorUInt32Swig_erase, METH_VARARGS, NULL}, + { "new_VectorUInt32Swig", _wrap_new_VectorUInt32Swig, METH_VARARGS, NULL}, + { "VectorUInt32Swig_push_back", _wrap_VectorUInt32Swig_push_back, METH_VARARGS, NULL}, + { "VectorUInt32Swig_front", _wrap_VectorUInt32Swig_front, METH_O, NULL}, + { "VectorUInt32Swig_back", _wrap_VectorUInt32Swig_back, METH_O, NULL}, + { "VectorUInt32Swig_assign", _wrap_VectorUInt32Swig_assign, METH_VARARGS, NULL}, + { "VectorUInt32Swig_resize", _wrap_VectorUInt32Swig_resize, METH_VARARGS, NULL}, + { "VectorUInt32Swig_insert", _wrap_VectorUInt32Swig_insert, METH_VARARGS, NULL}, + { "VectorUInt32Swig_reserve", _wrap_VectorUInt32Swig_reserve, METH_VARARGS, NULL}, + { "VectorUInt32Swig_capacity", _wrap_VectorUInt32Swig_capacity, METH_O, NULL}, + { "delete_VectorUInt32Swig", _wrap_delete_VectorUInt32Swig, METH_O, NULL}, + { "VectorUInt32Swig_swigregister", VectorUInt32Swig_swigregister, METH_O, NULL}, + { "VectorUInt32Swig_swiginit", VectorUInt32Swig_swiginit, METH_VARARGS, NULL}, { "new_EngineHashSwig", _wrap_new_EngineHashSwig, METH_VARARGS, NULL}, { "EngineHashSwig_getPublishedTime", _wrap_EngineHashSwig_getPublishedTime, METH_O, NULL}, { "EngineHashSwig_getUpdateAvailableTime", _wrap_EngineHashSwig_getUpdateAvailableTime, METH_O, NULL}, @@ -28601,6 +32112,7 @@ static PyMethodDef SwigMethods[] = { { "EngineHashSwig_refreshDataFromFile", _wrap_EngineHashSwig_refreshDataFromFile, METH_VARARGS, NULL}, { "EngineHashSwig_refreshDataFromMemory", _wrap_EngineHashSwig_refreshDataFromMemory, METH_VARARGS, NULL}, { "EngineHashSwig_process", _wrap_EngineHashSwig_process, METH_VARARGS, NULL}, + { "EngineHashSwig_getProfileIdsForPropertyValue", _wrap_EngineHashSwig_getProfileIdsForPropertyValue, METH_VARARGS, NULL}, { "EngineHashSwig_processBase", _wrap_EngineHashSwig_processBase, METH_VARARGS, NULL}, { "EngineHashSwig_processDeviceDetection", _wrap_EngineHashSwig_processDeviceDetection, METH_VARARGS, NULL}, { "delete_EngineHashSwig", _wrap_delete_EngineHashSwig, METH_O, NULL}, @@ -28706,10 +32218,12 @@ static swig_type_info _swigt__p_size_type = {"_p_size_type", "size_type *", 0, 0 static swig_type_info _swigt__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t = {"_p_std__allocatorT_std__pairT_std__string_const_std__string_t_t", "std::map< std::string,std::string >::allocator_type *|std::allocator< std::pair< std::string const,std::string > > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__allocatorT_std__string_t = {"_p_std__allocatorT_std__string_t", "std::vector< std::string >::allocator_type *|std::allocator< std::string > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__allocatorT_unsigned_char_t = {"_p_std__allocatorT_unsigned_char_t", "std::vector< unsigned char >::allocator_type *|std::allocator< unsigned char > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__allocatorT_unsigned_int_t = {"_p_std__allocatorT_unsigned_int_t", "std::vector< unsigned int >::allocator_type *|std::allocator< unsigned int > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__invalid_argument = {"_p_std__invalid_argument", "std::invalid_argument *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__lessT_std__string_t = {"_p_std__lessT_std__string_t", "std::less< std::string > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__mapT_std__string_std__string_t = {"_p_std__mapT_std__string_std__string_t", "std::map< std::string,std::string,std::less< std::string >,std::allocator< std::pair< std::string const,std::string > > > *|std::map< std::string,std::string > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string,std::allocator< std::string > > *|std::vector< std::string > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_uint32_t_t = {"_p_std__vectorT_uint32_t_t", "std::vector< uint32_t > *|std::vector< unsigned int,std::allocator< unsigned int > > *|std::vector< unsigned int > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_std__vectorT_uint8_t_t = {"_p_std__vectorT_uint8_t_t", "std::vector< uint8_t > *|std::vector< unsigned char,std::allocator< unsigned char > > *|std::vector< unsigned char > *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_swig__SwigPyIterator = {"_p_swig__SwigPyIterator", "swig::SwigPyIterator *", 0, 0, (void*)0, 0}; static swig_type_info _swigt__p_unsigned_char = {"_p_unsigned_char", "uint8_t *|uint_fast8_t *|uint_least8_t *|unsigned char *", 0, 0, (void*)0, 0}; @@ -28765,10 +32279,12 @@ static swig_type_info *swig_type_initial[] = { &_swigt__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t, &_swigt__p_std__allocatorT_std__string_t, &_swigt__p_std__allocatorT_unsigned_char_t, + &_swigt__p_std__allocatorT_unsigned_int_t, &_swigt__p_std__invalid_argument, &_swigt__p_std__lessT_std__string_t, &_swigt__p_std__mapT_std__string_std__string_t, &_swigt__p_std__vectorT_std__string_t, + &_swigt__p_std__vectorT_uint32_t_t, &_swigt__p_std__vectorT_uint8_t_t, &_swigt__p_swig__SwigPyIterator, &_swigt__p_unsigned_char, @@ -28824,10 +32340,12 @@ static swig_cast_info _swigc__p_size_type[] = { {&_swigt__p_size_type, 0, 0, 0} static swig_cast_info _swigc__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t[] = { {&_swigt__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__allocatorT_std__string_t[] = { {&_swigt__p_std__allocatorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__allocatorT_unsigned_char_t[] = { {&_swigt__p_std__allocatorT_unsigned_char_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__allocatorT_unsigned_int_t[] = { {&_swigt__p_std__allocatorT_unsigned_int_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__invalid_argument[] = { {&_swigt__p_std__invalid_argument, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__lessT_std__string_t[] = { {&_swigt__p_std__lessT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__mapT_std__string_std__string_t[] = { {&_swigt__p_std__mapT_std__string_std__string_t, 0, 0, 0}, {&_swigt__p_EvidenceBase, _p_EvidenceBaseTo_p_std__mapT_std__string_std__string_t, 0, 0}, {&_swigt__p_EvidenceDeviceDetection, _p_EvidenceDeviceDetectionTo_p_std__mapT_std__string_std__string_t, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = { {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_uint32_t_t[] = { {&_swigt__p_std__vectorT_uint32_t_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_std__vectorT_uint8_t_t[] = { {&_swigt__p_std__vectorT_uint8_t_t, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_swig__SwigPyIterator[] = { {&_swigt__p_swig__SwigPyIterator, 0, 0, 0},{0, 0, 0, 0}}; static swig_cast_info _swigc__p_unsigned_char[] = { {&_swigt__p_unsigned_char, 0, 0, 0},{0, 0, 0, 0}}; @@ -28883,10 +32401,12 @@ static swig_cast_info *swig_cast_initial[] = { _swigc__p_std__allocatorT_std__pairT_std__string_const_std__string_t_t, _swigc__p_std__allocatorT_std__string_t, _swigc__p_std__allocatorT_unsigned_char_t, + _swigc__p_std__allocatorT_unsigned_int_t, _swigc__p_std__invalid_argument, _swigc__p_std__lessT_std__string_t, _swigc__p_std__mapT_std__string_std__string_t, _swigc__p_std__vectorT_std__string_t, + _swigc__p_std__vectorT_uint32_t_t, _swigc__p_std__vectorT_uint8_t_t, _swigc__p_swig__SwigPyIterator, _swigc__p_unsigned_char, diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx index 61d05328..0d472453 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipi_python_wrap.cxx @@ -47460,263 +47460,13 @@ SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_2(PyObject *self, Py_ssiz } -SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_3(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineIpi *arg1 = 0 ; - EvidenceIpi *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; - void *argp1 = 0 ; - int res1 = 0 ; - void *argp2 = 0 ; - int res2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsIpi *result = 0 ; - - (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineIpi, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineIpiSwig_process" "', argument " "1"" of type '" "EngineIpi *""'"); - } - arg1 = reinterpret_cast< EngineIpi * >(argp1); - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_EvidenceIpi, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineIpiSwig_process" "', argument " "2"" of type '" "EvidenceIpi *""'"); - } - arg2 = reinterpret_cast< EvidenceIpi * >(argp2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineIpiSwig_process" "', argument " "3"" of type '" "int const *""'"); - } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineIpiSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); - { - try { - result = (ResultsIpi *)(arg1)->process(arg2,(int const *)arg3,arg4);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsIpi, SWIG_POINTER_OWN | 0 ); - return resultobj; -fail: - return NULL; -} - - -SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_4(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineIpi *arg1 = 0 ; - char *arg2 = 0 ; - int *arg3 = 0 ; - int arg4 ; - void *argp1 = 0 ; - int res1 = 0 ; - int res2 ; - char *buf2 = 0 ; - int alloc2 = 0 ; - void *argp3 = 0 ; - int res3 = 0 ; - int val4 ; - int ecode4 = 0 ; - ResultsIpi *result = 0 ; - - (void)self; - if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineIpi, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineIpiSwig_process" "', argument " "1"" of type '" "EngineIpi *""'"); - } - arg1 = reinterpret_cast< EngineIpi * >(argp1); - res2 = SWIG_AsCharPtrAndSize(swig_obj[1], &buf2, NULL, &alloc2); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineIpiSwig_process" "', argument " "2"" of type '" "char const *""'"); - } - arg2 = reinterpret_cast< char * >(buf2); - res3 = SWIG_ConvertPtr(swig_obj[2], &argp3,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res3)) { - SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "EngineIpiSwig_process" "', argument " "3"" of type '" "int const *""'"); - } - arg3 = reinterpret_cast< int * >(argp3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineIpiSwig_process" "', argument " "4"" of type '" "int""'"); - } - arg4 = static_cast< int >(val4); - { - try { - result = (ResultsIpi *)(arg1)->process((char const *)arg2,(int const *)arg3,arg4);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsIpi, SWIG_POINTER_OWN | 0 ); - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return resultobj; -fail: - if (alloc2 == SWIG_NEWOBJ) delete[] buf2; - return NULL; -} - - -SWIGINTERN PyObject *_wrap_EngineIpiSwig_process__SWIG_5(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { - PyObject *resultobj = 0; - EngineIpi *arg1 = 0 ; - unsigned char *arg2 = (unsigned char *)0 ; - long arg3 ; - fiftyoneDegreesIpType arg4 ; - int *arg5 = 0 ; - int arg6 ; - void *argp1 = 0 ; - int res1 = 0 ; - void *argp2 = 0 ; - int res2 = 0 ; - long val3 ; - int ecode3 = 0 ; - int val4 ; - int ecode4 = 0 ; - void *argp5 = 0 ; - int res5 = 0 ; - int val6 ; - int ecode6 = 0 ; - ResultsIpi *result = 0 ; - - (void)self; - if ((nobjs < 6) || (nobjs > 6)) SWIG_fail; - res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_EngineIpi, 0 | 0 ); - if (!SWIG_IsOK(res1)) { - SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "EngineIpiSwig_process" "', argument " "1"" of type '" "EngineIpi *""'"); - } - arg1 = reinterpret_cast< EngineIpi * >(argp1); - res2 = SWIG_ConvertPtr(swig_obj[1], &argp2,SWIGTYPE_p_unsigned_char, 0 | 0 ); - if (!SWIG_IsOK(res2)) { - SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "EngineIpiSwig_process" "', argument " "2"" of type '" "unsigned char []""'"); - } - arg2 = reinterpret_cast< unsigned char * >(argp2); - ecode3 = SWIG_AsVal_long(swig_obj[2], &val3); - if (!SWIG_IsOK(ecode3)) { - SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "EngineIpiSwig_process" "', argument " "3"" of type '" "long""'"); - } - arg3 = static_cast< long >(val3); - ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); - if (!SWIG_IsOK(ecode4)) { - SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "EngineIpiSwig_process" "', argument " "4"" of type '" "fiftyoneDegreesIpType""'"); - } - arg4 = static_cast< fiftyoneDegreesIpType >(val4); - res5 = SWIG_ConvertPtr(swig_obj[4], &argp5,SWIGTYPE_p_int, 0 | 0 ); - if (!SWIG_IsOK(res5)) { - SWIG_exception_fail(SWIG_ArgError(res5), "in method '" "EngineIpiSwig_process" "', argument " "5"" of type '" "int const *""'"); - } - arg5 = reinterpret_cast< int * >(argp5); - ecode6 = SWIG_AsVal_int(swig_obj[5], &val6); - if (!SWIG_IsOK(ecode6)) { - SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "EngineIpiSwig_process" "', argument " "6"" of type '" "int""'"); - } - arg6 = static_cast< int >(val6); - { - try { - result = (ResultsIpi *)(arg1)->process(arg2,arg3,arg4,(int const *)arg5,arg6);; - } - catch(FatalException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NotImplementedException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(InvalidPropertyException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(EvidenceException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(TooManyValuesException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(NoValuesAvailableException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(StatusCodeException& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(runtime_error& e) { - SWIG_exception(SWIG_RuntimeError, e.what()); - } - catch(invalid_argument& e) { - SWIG_exception(SWIG_ValueError, e.what()); - } - } - resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_ResultsIpi, SWIG_POINTER_OWN | 0 ); - return resultobj; -fail: - return NULL; -} - - SWIGINTERN PyObject *_wrap_EngineIpiSwig_process(PyObject *self, PyObject *args) { Py_ssize_t argc; - PyObject *argv[7] = { + PyObject *argv[5] = { 0 }; - if (!(argc = SWIG_Python_UnpackTuple(args, "EngineIpiSwig_process", 0, 6, argv))) SWIG_fail; + if (!(argc = SWIG_Python_UnpackTuple(args, "EngineIpiSwig_process", 0, 4, argv))) SWIG_fail; --argc; if (argc == 2) { int _v = 0; @@ -47771,102 +47521,13 @@ SWIGINTERN PyObject *_wrap_EngineIpiSwig_process(PyObject *self, PyObject *args) } } } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_EvidenceIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineIpiSwig_process__SWIG_3(self, argc, argv); - } - } - } - } - } - if (argc == 4) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[2], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineIpiSwig_process__SWIG_4(self, argc, argv); - } - } - } - } - } - if (argc == 6) { - int _v = 0; - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_EngineIpi, 0); - _v = SWIG_CheckState(res); - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[1], &vptr, SWIGTYPE_p_unsigned_char, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_long(argv[2], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - { - int res = SWIG_AsVal_int(argv[3], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - void *vptr = 0; - int res = SWIG_ConvertPtr(argv[4], &vptr, SWIGTYPE_p_int, 0); - _v = SWIG_CheckState(res); - if (_v) { - { - int res = SWIG_AsVal_int(argv[5], NULL); - _v = SWIG_CheckState(res); - } - if (_v) { - return _wrap_EngineIpiSwig_process__SWIG_5(self, argc, argv); - } - } - } - } - } - } - } fail: SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'EngineIpiSwig_process'.\n" " Possible C/C++ prototypes are:\n" " EngineIpi::process(EvidenceIpi *)\n" " EngineIpi::process(char const *)\n" - " EngineIpi::process(unsigned char [],long,fiftyoneDegreesIpType)\n" - " EngineIpi::process(EvidenceIpi *,int const *,int)\n" - " EngineIpi::process(char const *,int const *,int)\n" - " EngineIpi::process(unsigned char [],long,fiftyoneDegreesIpType,int const *,int)\n"); + " EngineIpi::process(unsigned char [],long,fiftyoneDegreesIpType)\n"); return 0; } From 23188d3f241b273476c7ff736ebcc2e471c2cd3a Mon Sep 17 00:00:00 2001 From: Automation51D <60396166+Automation51D@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:56:47 +0100 Subject: [PATCH 04/26] Merged Pull Request '#26 update-submodules/rc->rc: Update submodules' Co-authored-by: Automation51D <51DCI@51Degrees.com> --- ip-intelligence/ip-intelligence-data | 2 +- .../src/fiftyone_pipeline_core/js_templates | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ip-intelligence/ip-intelligence-data b/ip-intelligence/ip-intelligence-data index b0f0b6af..b8b3d7d4 160000 --- a/ip-intelligence/ip-intelligence-data +++ b/ip-intelligence/ip-intelligence-data @@ -1 +1 @@ -Subproject commit b0f0b6afe4fe741bc93a7bd6a7a2ad1058dd6e71 +Subproject commit b8b3d7d4e7747a0412281b9b48e0144b2ef13fcc diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates index d3f96049..c773ce22 160000 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/js_templates @@ -1 +1 @@ -Subproject commit d3f96049bde389951c79638047b8ea80dfdd5cdf +Subproject commit c773ce221ccec6fcb67fbdd5ad2175a66fab11d6 From 6732e18af0789559991b8d0d8f4f37666e77a50b Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:24:48 +0000 Subject: [PATCH 05/26] Read the data files from the data repository in cxx/data (#28) --- .gitmodules | 3 - AGENTS.md | 18 ++++- README.md | 23 ++++++ ci/packages.json | 2 - cxx | 2 +- device-detection/ci/fetch-assets.ps1 | 12 ++- .../examples/gettingstarted.py | 2 +- .../example_utils.py | 76 +++++++++++++++++-- .../onpremise/datafileupdate_console.py | 5 +- .../onpremise/gettingstarted_console.py | 6 +- .../onpremise/gettingstarted_web/app.py | 5 +- .../gettingstarted_web/templates/index.html | 4 +- .../onpremise/match_metrics.py | 5 +- .../onpremise/metadata_console.py | 5 +- .../onpremise/offlineprocessing.py | 5 +- .../onpremise/performance.py | 2 +- .../onpremise/shareusage.py | 5 +- .../onpremise/useragentclienthints_web/app.py | 5 +- .../tests/test_devicedetection.py | 4 +- .../tests/test_properties.py | 2 +- .../tests/test_swig.py | 2 +- device-detection/readme.md | 19 +++-- ip-intelligence/ci/fetch-assets.ps1 | 8 ++ .../readme.md | 13 ++-- .../example_utils.py | 72 +++++++++++++++--- .../onpremise/compare_console.py | 10 +-- .../onpremise/countries_console.py | 5 +- .../onpremise/gettingstarted_console.py | 13 ++-- .../onpremise/gettingstarted_web/app.py | 8 +- .../onpremise/metadata_console.py | 5 +- .../onpremise/mixed_console.py | 19 +++-- .../onpremise/offlineprocessing.py | 8 +- .../onpremise/performance_console.py | 6 +- .../tests/test_examples.py | 12 +-- .../readme.md | 19 +++-- .../tests/constants.py | 11 +-- .../tests/test_ipintelligence.py | 39 +++++----- .../example_constants.py | 13 ++-- .../.gitignore | 4 +- .../setup.py | 15 ++-- .../resources.py | 15 ++-- .../tests/test_translation.py | 15 ++++ ip-intelligence/ip-intelligence-data | 1 - ip-intelligence/readme.md | 43 ++++++----- 44 files changed, 388 insertions(+), 178 deletions(-) delete mode 160000 ip-intelligence/ip-intelligence-data diff --git a/.gitmodules b/.gitmodules index f0d05919..af600379 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule "pipeline/owid-python"] path = pipeline/owid-python url = https://github.com/SWAN-community/owid-python.git -[submodule "ip-intelligence/ip-intelligence-data"] - path = ip-intelligence/ip-intelligence-data - url = ../ip-intelligence-data [submodule "cxx"] path = cxx url = ../51degrees-cxx diff --git a/AGENTS.md b/AGENTS.md index 8dececdf..11855b13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,10 +39,20 @@ so a pipeline change is tested with every product in the same run. ## Submodules -The C/C++ engines and the data files are submodules, listed in the root -`.gitmodules` at their paths under each product folder. The IP Intelligence -on-premise package must compile its own copy of the common C library with -large data file support; the Device Detection one without it. +The C/C++ engines are the `cxx` submodule, listed in the root `.gitmodules`. +The data files come through cxx's own `data` submodule, at +`cxx/data/device-detection` and `cxx/data/ip-intelligence`, where each free +file is a `.link` that `cxx/data/get-data-files.ps1` or +`cxx/data/get-data-files.sh` downloads. Each product's `ci/fetch-assets.ps1` +gets the files its tests need, and the examples' `example_utils.py` runs the +script for a file that is missing. The IP Intelligence on-premise package must +compile its own copy of the common C library with large data file support; +the Device Detection one without it. + +A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside +`cxx`, because git reports any change there as a change to `cxx` itself. A +cxx update therefore releases no package, even when it brings new translations +in `cxx/data/ip-intelligence`. ## History diff --git a/README.md b/README.md index 3110be9c..2303b4fe 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,29 @@ Clone with the submodules: git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git ``` +The data files the examples and tests use are not stored in a repository. The +[data](https://github.com/51Degrees/data) repository, which is the `cxx/data` +submodule, holds a `.link` file for each free one, where the file goes, and +two scripts that download them. From the root of this repository, run + +``` +sh cxx/data/get-data-files.sh +``` + +or, in PowerShell, + +``` +./cxx/data/get-data-files.ps1 +``` + +An example whose file is missing runs the script for that file itself. + +Git leaves the folder of a removed submodule on disk. A clone made before the +data files moved to `cxx/data` still has an `ip-intelligence/ip-intelligence-data` +folder, and keeps `cxx/device-detection-data` and `cxx/ip-intelligence-data` +once cxx removes them. Delete those folders, so that nothing finds an old copy +of a data file in them. + Each product's README says what else it needs, such as a C/C++ compiler for the on-premise engines and the data files. diff --git a/ci/packages.json b/ci/packages.json index 873df967..deb06ce9 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -140,7 +140,6 @@ "Name": "fiftyone_ipintelligence_onpremise", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise", - "ExtraPaths": ["ip-intelligence/ip-intelligence-data"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone" @@ -153,7 +152,6 @@ "Name": "fiftyone_ipintelligence_translation", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_translation", - "ExtraPaths": ["ip-intelligence/ip-intelligence-data"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_translation" diff --git a/cxx b/cxx index 1bec7b57..e866f6d3 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 1bec7b570575e2e180b52435599bef9b03e35ad8 +Subproject commit e866f6d3cea936587790f87967eff7863778b30e diff --git a/device-detection/ci/fetch-assets.ps1 b/device-detection/ci/fetch-assets.ps1 index c551f8f9..45cb2875 100644 --- a/device-detection/ci/fetch-assets.ps1 +++ b/device-detection/ci/fetch-assets.ps1 @@ -11,7 +11,8 @@ if ($env:GITHUB_JOB -eq "PreBuild") { exit 0 } -$deviceDetectionData = "$PSScriptRoot/../../cxx/device-detection-data" +$data = "$PSScriptRoot/../../cxx/data" +$deviceDetectionData = "$data/device-detection" $assets = '51Degrees-LiteV4.1.hash', '20000 Evidence Records.yml', '20000 User Agents.csv' if ($DeviceDetection) { @@ -29,3 +30,12 @@ foreach ($asset in $assets) { default {New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$_" -Path "$deviceDetectionData/$_"} } } + +# Downloads any other file the data repository links to in the folder. The +# script keeps a file that is already there, so the assets linked in above +# are not downloaded again. +$global:LASTEXITCODE = 0 +& "$data/get-data-files.ps1" device-detection +if ($LASTEXITCODE -ne 0) { + throw "Not every data file in $deviceDetectionData was downloaded" +} diff --git a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py index 994d736f..cf3a7b7f 100644 --- a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py +++ b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py @@ -24,7 +24,7 @@ # First create the device detection pipeline with the desired settings. -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" resource_key = "!!YOUR_RESOURCE_KEY!!" # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index d36c35ea..05260f95 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -23,6 +23,8 @@ from datetime import datetime, timedelta from threading import Event, Thread import os +import shlex +import subprocess from pathlib import Path @@ -51,6 +53,15 @@ class ExampleUtils: # file named in its configuration. DATA_FILE_ENV_VAR = "51DEGREES_DD_PATH" + # The data repository's device detection folder, from the root of a + # checkout of this repository. Each file the data repository links to + # belongs in this folder, beside its .link file. + DATA_FOLDER = os.path.join("cxx", "data", "device-detection") + + # Sent with each download, so the downloads these examples make can be + # counted. + DOWNLOAD_SOURCE = "examples-51degrees-python" + @staticmethod def get_resource_key_from_config(config): key = "" @@ -183,12 +194,29 @@ def __get_env_variable(name): else: return "" - # Find the specified filename within the specified directory. - # If no directory is specified, the working directory is used. - # If the file cannot be found, the algorithm will move to the - # parent directory and repeat the process. + # Find the specified file. A path to a file that exists is returned as + # it is. Next, the data repository's folder is checked in the working + # directory and in each directory above it, and a file the data + # repository links to is downloaded there when it is missing. Otherwise + # the working directory is searched for a file with the same name, then + # each parent directory in turn. None is returned when nothing is found. @staticmethod def find_file(file_name): + path = Path(file_name).absolute() + if path.is_file(): + return str(path.resolve()) + + # The data repository's copy is preferred to a file with the same + # name elsewhere in the tree, which may be out of date or a Git LFS + # pointer. + current = Path(os.getcwd()).absolute() + for root in [current, *current.parents]: + candidate = root / ExampleUtils.DATA_FOLDER / path.name + if candidate.is_file(): + return str(candidate) + if candidate.with_name(candidate.name + ".link").is_file(): + return ExampleUtils.__download(str(candidate)) + stop_event = Event() result = [None] # Start the file system search as a separate task. @@ -235,6 +263,39 @@ def __find_file(file_name, stop_event, result, search_dir=None): if search_dir.parent != search_dir and not stop_event.is_set(): ExampleUtils.__find_file(file_name, stop_event, result, search_dir.parent) + # Download a file the data repository links to, by running the data + # repository's own script for that file, and return its path, or None + # when it could not be downloaded. The data repository is the folder + # above the one that holds the file's .link. + @staticmethod + def __download(path): + name = os.path.basename(path) + data = os.path.dirname(os.path.dirname(path)) + if os.name == "nt": + command = [ + "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name, + "-Source", ExampleUtils.DOWNLOAD_SOURCE] + text = subprocess.list2cmdline(command) + else: + command = [ + "sh", os.path.join(data, "get-data-files.sh"), + "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + text = shlex.join(command) + + # Flushed first, because the script writes to the same output. + print(f"{name} is missing, so it is being downloaded by running\n" + f"{text}", flush=True) + try: + subprocess.run(command, check=False) + except OSError: + pass + if os.path.isfile(path): + return path + print(f"{name} could not be downloaded. Run this command to try " + f"again.\n{text}") + return None + @staticmethod def get_data_file_date(engine): date = engine.engine.getPublishedTime() @@ -272,9 +333,10 @@ def check_data_file(pipeline, logger): f"that is more than '{ExampleUtils.DATA_FILE_AGE_WARNING}' days " + "old. A more recent data file may be needed to " + "correctly detect the latest devices, browsers, " + - "etc. The latest lite data file is available from " + - "the device-detection-data repository on GitHub " + - "https://github.com/51Degrees/device-detection-data. " + + "etc. Get the latest lite data file by running " + + "cxx/data/get-data-files.sh -u 51Degrees-LiteV4.1.hash, " + + "or cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash " + + "-Update on Windows. " + "Find out about the Enterprise data file, which " + "includes automatic updates, on our pricing page: " + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index bf9ff7a3..d5fd2228 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -53,9 +53,8 @@ # # ## Lite Data File # Lite data files (free-to-use, limited capabilities, no license key required) are created roughly -# once a month and cannot be updated using auto-update, they may be downloaded from -# [Github](https://github.com/51Degrees/device-detection-data) and are included with -# source distributions of this software. +# once a month and cannot be updated using auto-update, they may be downloaded with the +# scripts in the [data](https://github.com/51Degrees/data) repository on GitHub. # # # Update on Start-Up # You can configure the pipeline builder to download an Enterprise data file on start-up. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 9c695f43..f49c5bc0 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -152,9 +152,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection " + - "data file. Make sure the device-detection-data " + - "submodule has been updated by running " + - "`git submodule update --recursive`.") + "data file. Make sure the data submodule is there, " + + "by running `git submodule update --init --recursive`, " + + "and run this example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index c2ec0e2b..dee9b99b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -246,8 +246,9 @@ def build_config(): if foundDataFile == False: raise Exception("Failed to find a device detection data file matching " + f"'{dataFile}'. If using the lite file, then make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`. Otherwise, ensure that the filename " + + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file. Otherwise, ensure that the filename " + "is correct in config.json, or set the environment variable " + f"'{ExampleUtils.DATA_FILE_ENV_VAR}' to the path of the data file.") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html index 8106f4cc..d26ab8da 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html @@ -30,8 +30,8 @@
    WARNING: This example is using a data file that is more than {{ utils.DATA_FILE_AGE_WARNING }} days old. A more recent data file may be needed to correctly detect the latest devices, - browsers, etc. The latest lite data file is available from the - device-detection-data + browsers, etc. Get the latest lite data file with the scripts in the + data repository on GitHub. Find out about the Enterprise data file, which includes automatic daily updates, on our pricing page.
    diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index f9f254f8..5daeb284 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -189,8 +189,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection data file. Make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`.") + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 1e5e0929..ed974ae6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -129,8 +129,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection data file. Make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`.") + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 2487f3e9..80ca344f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -211,8 +211,9 @@ def main(argv): else: logger.log("error", "Failed to find a device detection data file. Make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`.") + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index 665cdabf..271ce134 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -145,7 +145,7 @@ def run(data_file, skip = False): if __name__ == "__main__": ap = argparse.ArgumentParser(description='Run detection benchmark.') ap.add_argument('-d', '--data_file', default='', help='Path to data file') - ap.add_argument('-u', '--user_agents_file', default='../../cxx/device-detection-data/20000 User Agents.csv', help='Path to user agents evidence file') + ap.add_argument('-u', '--user_agents_file', default='../../cxx/data/device-detection/20000 User Agents.csv', help='Path to user agents evidence file') ap.add_argument('-j', '--json_output', default='', help='Output results in JSON format') args = ap.parse_args() if args.data_file == "": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py index 5d808bf4..6cbbbfd7 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py @@ -184,8 +184,9 @@ def main(argv): ShareUsageExample().run(input) else: print("ERROR: Failed to find the evidence file. Make sure the " - "device-detection-data submodule has been updated by running " - "`git submodule update --recursive`.") + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file.") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 6d10a1a7..570ab57e 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -148,8 +148,9 @@ def build_config(): if foundDataFile == False: raise Exception("Failed to find a device detection data file matching " + f"'{dataFile}'. If using the lite file, then make sure the " + - "device-detection-data submodule has been updated by running " + - "`git submodule update --recursive`. Otherwise, ensure that the filename " + + "data submodule is there, by running " + + "`git submodule update --init --recursive`, and run this " + + "example again to download the file. Otherwise, ensure that the filename " + "is correct in config.json.") return config diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index 3cd885ce..135cc69d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -36,7 +36,7 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_engines.lru_cache import LRUEngineCache -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" @@ -46,7 +46,7 @@ # User agents file -with open('../../cxx/device-detection-data/20000 User Agents.csv', newline='') as file: +with open('../../cxx/data/device-detection/20000 User Agents.csv', newline='') as file: reader = csv.reader(file) user_agents = list(reader) no_user_agents = len(user_agents) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py index def20c1c..0d9b6435 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py @@ -28,7 +28,7 @@ from fiftyone_devicedetection_shared.utils import * from fiftyone_devicedetection_onpremise import constants -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile" "/11D167 Safari/9537.53") diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py index 04b4ee50..0b4bea5a 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py @@ -28,7 +28,7 @@ from fiftyone_devicedetection_onpremise.swig_data import SwigData from fiftyone_devicedetection_shared.utils import * -data_file = "../../cxx/device-detection-data/51Degrees-LiteV4.1.hash" +data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" # Create a simple pipeline to access the engine with and process it with flow data deviceDetectionOnPremiseEngine = DeviceDetectionOnPremise( diff --git a/device-detection/readme.md b/device-detection/readme.md index 379b1ed1..379be178 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -43,22 +43,25 @@ to use the Cloud API. You can create resource keys using our #### On-Premise In order to perform device detection on-premise, you will need to use a 51Degrees data file. -This repository includes a free, 'lite' file in the 'device-detection-data' sub-module that has a -significantly reduced set of properties. To obtain a file with a more complete set of device -properties see the [51Degrees website](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=on-premise). If you want to use the lite -file, you will need to install [GitLFS](https://git-lfs.github.com/): +A free, 'lite' file is available, with a significantly reduced set of properties. To obtain a +file with a more complete set of device properties see the [51Degrees website](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=on-premise). + +The lite file is not stored in a repository. The [data](https://github.com/51Degrees/data) +repository, which is the `cxx/data` submodule, links to it from +`cxx/data/device-detection`. Download it from the root of this repository by running: ``` -sudo apt-get install git-lfs -git lfs install +sh cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash ``` -Then, navigate to 'src/fiftyone_devicedetection_onpremise/cxx/device-detection-data' and execute: +or, in PowerShell: ``` -git lfs pull +./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash ``` +The examples run this for themselves when the file is missing. + ## Folders * `fiftyone_devicedetection` - references both cloud and on-premise packages, contains generic Device-Detection Pipeline Builder. diff --git a/ip-intelligence/ci/fetch-assets.ps1 b/ip-intelligence/ci/fetch-assets.ps1 index 00e2bd91..a26332bc 100644 --- a/ip-intelligence/ci/fetch-assets.ps1 +++ b/ip-intelligence/ci/fetch-assets.ps1 @@ -29,6 +29,14 @@ foreach ($asset in $assets) { } } +# The ASN file is not an asset, so the data repository's script downloads it +# beside its link in cxx/data/ip-intelligence, where the tests look for it. +$global:LASTEXITCODE = 0 +& "$PSScriptRoot/../../cxx/data/get-data-files.ps1" 51Degrees-IPIV4AsnIpiV41.ipi +if ($LASTEXITCODE -ne 0) { + throw "The ASN data file was not downloaded" +} + # If the Enterprise file was downloaded, prefer it for tests by exposing # it through the environment variable used by the test suites. if ($IpIntelligenceUrl) { diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index af0db8c7..d9542f6a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -34,11 +34,12 @@ python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_web ## On-premise examples Require a `.ipi` data file in the format the engine reads (see -[Data files](../readme.md#data-files)). The -`ip-intelligence-data` submodule includes a freely available network-only -file (`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to; it -only contains the ASN/network properties, so location and registration -properties are reported as "not present in the data file". +[Data files](../readme.md#data-files)). The data repository, at +`cxx/data`, links to a freely available network-only file +(`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to, and an +example downloads it when it is missing. It only contains the ASN/network +properties, so location and registration properties are reported as "not +present in the data file". For the full set of properties (location, registration, etc.), a more complete data file is required - to obtain one for testing please @@ -50,7 +51,7 @@ the first command line argument. - `onpremise/metadata_console.py` - list the properties available from the data file. - `onpremise/offlineprocessing.py` - batch process the evidence file - from the ip-intelligence-data repository. + from the data repository. - `onpremise/performance_console.py` - benchmark the number of IP lookups performed per second. - `onpremise/compare_console.py` - compare the location returned for each diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index 56e3d783..81eb5829 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -22,6 +22,8 @@ from datetime import datetime, timedelta import os +import shlex +import subprocess from pathlib import Path @@ -43,6 +45,15 @@ class ExampleUtils: # to use when running examples. DATA_FILE_ENV_VAR = "IPINTELLIGENCEDATAFILE" + # The data repository's IP intelligence folder, from the root of a + # checkout of this repository. Each file the data repository links to + # belongs in this folder, beside its .link file. + DATA_FOLDER = os.path.join("cxx", "data", "ip-intelligence") + + # Sent with each download, so the downloads these examples make can be + # counted. + DOWNLOAD_SOURCE = "examples-51degrees-python" + @staticmethod def get_human_readable(ip_data, property): try: @@ -112,8 +123,9 @@ def find_file(file_name): """! Find the specified file. First checks the data file environment variable, then searches up the directory tree from the current - working directory looking in 'assets' and 'ip-intelligence-data' - folders. + working directory looking in 'assets' and the data repository's + folder. A file the data repository links to is downloaded there + when it is missing. """ env = os.environ.get(ExampleUtils.DATA_FILE_ENV_VAR) @@ -123,15 +135,54 @@ def find_file(file_name): current = Path(os.getcwd()).resolve() roots = [current, *current.parents] for root in roots[:8]: - for sub_dir in [ - ".", - "assets", - "ip-intelligence-data", - os.path.join("cxx", "ip-intelligence-data")]: + for sub_dir in [".", "assets", ExampleUtils.DATA_FOLDER]: candidate = os.path.join(str(root), sub_dir, file_name) if os.path.isfile(candidate): return candidate + for root in roots[:8]: + candidate = os.path.join( + str(root), ExampleUtils.DATA_FOLDER, file_name) + if os.path.isfile(candidate + ".link"): + return ExampleUtils.__download(candidate) + + return None + + @staticmethod + def __download(path): + + """! + Download a file the data repository links to, by running the data + repository's own script for that file, and return its path, or None + when it could not be downloaded. The data repository is the folder + above the one that holds the file's .link. + """ + + name = os.path.basename(path) + data = os.path.dirname(os.path.dirname(path)) + if os.name == "nt": + command = [ + "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name, + "-Source", ExampleUtils.DOWNLOAD_SOURCE] + text = subprocess.list2cmdline(command) + else: + command = [ + "sh", os.path.join(data, "get-data-files.sh"), + "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + text = shlex.join(command) + + # Flushed first, because the script writes to the same output. + print(f"{name} is missing, so it is being downloaded by running\n" + f"{text}", flush=True) + try: + subprocess.run(command, check=False) + except OSError: + pass + if os.path.isfile(path): + return path + print(f"{name} could not be downloaded. Run this command to try " + f"again.\n{text}") return None @staticmethod @@ -167,9 +218,10 @@ def check_data_file(pipeline, logger): f"This data file is more than " f"{ExampleUtils.DATA_FILE_AGE_WARNING} days old. A more " f"recent data file may be needed to correctly resolve " - f"the latest IP addresses. The latest data file is " - f"available from the ip-intelligence-data repository on " - f"GitHub https://github.com/51Degrees/ip-intelligence-data.") + f"the latest IP addresses. Get the latest free data files " + f"by running cxx/data/get-data-files.sh -u ip-intelligence, " + f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " + f"on Windows.") if product == "Lite": logger.log("warning", diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index d0479c2e..062e24da 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -30,8 +30,8 @@ # an IP-to-location service compare to real world information, especially # when evaluating different solutions. # -# The input CSV (geoip_comparison_evidence.csv in the ip-intelligence-data -# repository) provides these fields: +# The input CSV (geoip_comparison_evidence.csv in the data repository's +# ip-intelligence folder) provides these fields: # DateTimeUtc, Latitude, Longitude, Ip, AddressFamily, Continent, Country # # For each record the engine is used to obtain the latitude and longitude @@ -56,7 +56,7 @@ from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME -# Name of the truth CSV file in the ip-intelligence-data repository. +# Name of the truth CSV file in the data repository. GEOIP_COMPARISON_EVIDENCE_FILE_NAME = "geoip_comparison_evidence.csv" # Mean radius of the earth in kilometres, used for the distance calculation. @@ -184,8 +184,8 @@ def main(argv): else: logger.log("error", "Failed to find a data file or truth file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`. To obtain an enterprise " + "data submodule is there, by running " + "`git submodule update --init --recursive`. To obtain an enterprise " "data file for testing, please contact us: " "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=data-file-required") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index 35266657..6d26ce73 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -150,8 +150,9 @@ def main(argv): else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`, or supply the path to a " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " "data file as the first command line argument.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 0f211919..f5ad8cdc 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -32,10 +32,10 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example uses a local IP intelligence data file (.ipi). The -# ip-intelligence-data submodule provides a freely available data file; -# a more complete data file can be supplied as the first command line -# argument. +# This example uses a local IP intelligence data file (.ipi). The data +# repository links to a freely available data file, which the example +# downloads when it is missing. A more complete data file can be supplied +# as the first command line argument. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -170,8 +170,9 @@ def main(argv): else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`, or supply the path to a " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " "data file as the first command line argument.") if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index 49bc8b72..debc4a1b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -133,9 +133,11 @@ def main(data_file = None): data_file = ExampleUtils.find_file(LITE_DATAFILE_NAME) if not data_file: raise Exception( - "Failed to find an IP intelligence data file. Download the " - "Lite file from https://51ddatafiles.blob.core.windows.net/" - "enterpriseipi/51Degrees-LiteIpiV41.ipi.gz") + "Failed to find an IP intelligence data file. Make sure the " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " + "data file as the first command line argument.") build_pipeline(data_file) app.run(debug=False) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index e10df504..ec2be8a1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -66,9 +66,8 @@ def main(argv): MetadataConsole().run(data_file, logger, print) else: logger.log("error", - "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`. To obtain an enterprise " + "Failed to find an IP intelligence data file. Supply its path " + "as the first command line argument. To obtain an enterprise " "data file for testing, please contact us: " "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=data-file-required") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 11805592..07839ab1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -48,8 +48,8 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME -# Default name of the device detection data file (the lite file included in -# the device-detection-data repository). +# Default name of the device detection data file (the lite file the data +# repository links to). HASH_DATAFILE_NAME = "51Degrees-LiteV4.1.hash" # Combined evidence: a User-Agent for device detection and an IP address for @@ -199,14 +199,19 @@ def main(argv): else: if not device_file: logger.log("error", - "Failed to find a device detection data file. Make sure the " - "device-detection-data submodule has been updated, or supply " - "the path as the first command line argument.") + "Failed to find a device detection data file. Supply its " + "path as the first command line argument. Get the free Lite " + "file by running " + "cxx/data/get-data-files.sh 51Degrees-LiteV4.1.hash, or " + "cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash on " + "Windows, which saves it in cxx/data/device-detection.") if not ip_file: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " - "ip-intelligence-data submodule has been updated, or supply " - "the path as the second command line argument.") + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path as " + "the second command line argument.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index a32c11cb..9c7109cc 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -23,8 +23,8 @@ ## @example onpremise/offlineprocessing.py # Provides an example of processing a YAML file containing evidence for # IP intelligence. There are 20,000 IP addresses in the supplied file of -# evidence records ('evidence.yml' in the ip-intelligence-data -# repository). +# evidence records ('evidence.yml' in the data repository's +# ip-intelligence folder). # # We create a pipeline using the on-premise engine, deserialize the # YAML evidence records, pass each entry to the pipeline and write the @@ -120,8 +120,8 @@ def main(argv): else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`.") + "data submodule is there, by running " + "`git submodule update --init --recursive`.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 7ca71715..310bd727 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -25,7 +25,7 @@ # # This example measures the performance of the on-premise IP intelligence # engine. It processes the IP addresses from the evidence file supplied in -# the ip-intelligence-data repository and reports the number of IP lookups +# the data repository and reports the number of IP lookups # performed per second. # # Requesting fewer properties reduces processing time. If you do not specify @@ -225,8 +225,8 @@ def main(argv): else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " - "ip-intelligence-data submodule has been updated by running " - "`git submodule update --recursive`. To obtain an enterprise " + "data submodule is there, by running " + "`git submodule update --init --recursive`. To obtain an enterprise " "data file for testing, please contact us: " "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-performance_console.py&utm_term=data-file-required") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index b0e359a3..a2d17c30 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -29,10 +29,10 @@ from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME -# The tests run against the freely available data file committed to the -# ip-intelligence-data submodule, so they only rely on properties that the -# free data file provides (location properties such as Latitude, Longitude -# and Areas are not present in it). +# The tests run against the freely available ASN data file the data +# repository links to, so they only rely on properties that the free data +# file provides (location properties such as Latitude, Longitude and Areas +# are not present in it). data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = ExampleUtils.find_file(EVIDENCE_FILE_NAME) truth_file = ExampleUtils.find_file("geoip_comparison_evidence.csv") @@ -61,8 +61,8 @@ class ExampleTests(unittest.TestCase): @unittest.skipUnless( bool(data_file), "No IP intelligence data file available") def test_onpremise_datafile_loads(self): - # Verify the freely available data file committed to the - # ip-intelligence-data submodule loads and processes correctly. + # Verify the freely available ASN data file loads and processes + # correctly. from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder pipeline = IpIntelligencePipelineBuilder( data_file_path=data_file, diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index b7eb9ac3..65b11d8e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -28,12 +28,17 @@ python setup.py build_clib build_ext ## Data files -The on-premise engine requires a `.ipi` data file. A Lite data file can be -downloaded from -[51Degrees](https://51ddatafiles.blob.core.windows.net/enterpriseipi/51Degrees-LiteIpiV41.ipi.gz), -see also the [ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) -repository which is included as a nested submodule and contains evidence -files for testing. +The on-premise engine requires a `.ipi` data file. The free Lite data file, +`51Degrees-LiteV41.ipi`, is linked from the +[data](https://github.com/51Degrees/data) repository, which is included as +the nested submodule `cxx/data` and contains evidence files for testing. From +the root of a checkout of +[51degrees-python](https://github.com/51Degrees/51degrees-python), download it +by running `sh cxx/data/get-data-files.sh 51Degrees-LiteV41.ipi`, or +`./cxx/data/get-data-files.ps1 51Degrees-LiteV41.ipi` in PowerShell. Without a +checkout, download it from +https://distributor.51degrees.com/api/v2/free/ipi-lite-v41, which sends it +gzip compressed. ## Usage @@ -41,7 +46,7 @@ files for testing. from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise_pipelinebuilder import IpIntelligenceOnPremisePipelineBuilder pipeline = IpIntelligenceOnPremisePipelineBuilder( - data_file_path="51Degrees-LiteIpiV41.ipi", + data_file_path="51Degrees-LiteV41.ipi", licence_keys="", usage_sharing=False).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py index 6f9123f3..2c5180f8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/constants.py @@ -42,11 +42,12 @@ def get_data_file_path(): os.path.dirname(__file__), "..", "..", "assets", "51Degrees-LiteIpiV41.ipi") - # The ASN data file committed to the repository's ip-intelligence-data - # submodule. Unlike the Lite and Enterprise files it needs no download - # or secret, so it is always present wherever the engine can be built. + # The free ASN data file, which ci/fetch-assets.ps1 downloads with the + # data repository's script, beside its link in cxx/data/ip-intelligence. + # Unlike the Enterprise file it needs no secret. @staticmethod - def get_committed_data_file_path(): + def get_asn_data_file_path(): return os.path.join( os.path.dirname(__file__), - "..", "..", "ip-intelligence-data", ASN_DATAFILE_NAME) + "..", "..", "..", "cxx", "data", "ip-intelligence", + ASN_DATAFILE_NAME) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py index 13f5d042..df9f835e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py @@ -171,42 +171,41 @@ def test_missing_licence_key(self): self.assertIn("licence key is required", str(context.exception)) -class CommittedDataFileTests(unittest.TestCase): +class AsnDataFileTests(unittest.TestCase): """! - The native engine accepts exactly one data file format version, so the - cxx (51degrees-cxx) pin and the ip-intelligence-data pin must agree. - They once did not: the engine moved to format 4.6 while the data pin - still held the 4.5 ASN file. These tests open the ASN file committed to - the data submodule, which needs no download or secret, so pins that - disagree fail here on any machine. - - They do not prove the engine reads the data files 51Degrees publishes. - That is IpIntelligenceOnPremiseTests above, against the Lite or - Enterprise file CI downloads; it skips when neither is present. + The native engine accepts exactly one data file format version. These + tests open the free ASN data file. The Distributor always serves the + latest one published, and it needs no secret, so an engine that cannot + read the format 51Degrees publishes fails here on any machine. + + IpIntelligenceOnPremiseTests above runs against the Lite or Enterprise + file CI downloads, and skips when neither is present. test_onpremise_datafile_loads in the examples package also opens the ASN file, but it skips silently when the file is not found and it runs after the examples build, so it is not relied on here. """ - def test_committed_data_file_present(self): + def test_asn_data_file_present(self): """! - The committed data file exists. If it does not, the - ip-intelligence-data submodule was not checked out, and the test + The ASN data file has been downloaded. If it has not, the test below would prove nothing. """ self.assertTrue( - os.path.isfile(TestConstants.get_committed_data_file_path()), - "Run 'git submodule update --init --recursive'.") + os.path.isfile(TestConstants.get_asn_data_file_path()), + "Download it from the repository root by running " + "'sh cxx/data/get-data-files.sh 51Degrees-IPIV4AsnIpiV41.ipi', " + "or './cxx/data/get-data-files.ps1 51Degrees-IPIV4AsnIpiV41.ipi' " + "in PowerShell.") - def test_committed_data_file_opens(self): + def test_asn_data_file_opens(self): """! - The engine opens the committed data file, so it reads that file's - format version, and returns its properties for an IPv4 address. + The engine opens the ASN data file, so it reads that file's format + version, and returns its properties for an IPv4 address. """ engine = IpIntelligenceOnPremise( - data_file_path=TestConstants.get_committed_data_file_path(), + data_file_path=TestConstants.get_asn_data_file_path(), licence_keys="") pipeline = PipelineBuilder().add(engine).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py index e11dd421..35945298 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/src/fiftyone_ipintelligence_shared/example_constants.py @@ -38,12 +38,13 @@ # Note: the .NET examples' Constants.cs still uses the legacy name # "51Degrees-EnterpriseIpiV41.ipi", but the published enterprise/ASN data # files use the "IPIV4" infix (cf. 51Degrees-IPIV4AsnIpiV41.ipi in the -# ip-intelligence-data repo), so the current name is used here. +# data repository), so the current name is used here. ENTERPRISE_DATAFILE_NAME = '51Degrees-IPIV4EnterpriseIpiV41.ipi' -LITE_DATAFILE_NAME = '51Degrees-LiteIpiV41.ipi' -# The ASN data file is the freely available data file committed -# to the ip-intelligence-data submodule. It only contains the network -# (ASN) properties, but is useful for testing the on-premise engine against -# the current data format without an enterprise data file. +# The free Lite data file, named as the data repository links to it. +LITE_DATAFILE_NAME = '51Degrees-LiteV41.ipi' +# The ASN data file is a freely available data file the data repository +# links to. It only contains the network (ASN) properties, but is useful for +# testing the on-premise engine against the current data format without an +# enterprise data file. ASN_DATAFILE_NAME = '51Degrees-IPIV4AsnIpiV41.ipi' EVIDENCE_FILE_NAME = 'evidence.yml' diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore b/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore index 36ed5ddc..c39739ff 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore +++ b/ip-intelligence/fiftyone_ipintelligence_translation/.gitignore @@ -1,3 +1,3 @@ -# Translation YAML staged from the ip-intelligence-data submodule at build -# time (see setup.py stage_translations). Not committed; generated on build. +# Translation YAML staged from cxx/data/ip-intelligence at build time (see +# setup.py stage_translations). Not committed; generated on build. src/fiftyone_ipintelligence_translation/translations/ diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py index a29e4097..3f06c4a4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py @@ -40,14 +40,17 @@ def read(file_name): def stage_translations(): - """Copy the translation YAML files shipped in the ip-intelligence-data - submodule into the package so they are included in the built distribution. - This mirrors how the .NET project embeds the same files as resources. If the - submodule is not present the package falls back to reading it at run time, - so any failure here is ignored.""" + """Copy the translation YAML files from the data repository, the + cxx/data submodule, into the package so they are included in the built + distribution. This mirrors how the .NET project embeds the same files as + resources. A build from a source distribution has no cxx folder, but its + package already holds the files, so a missing folder is skipped. Any + failure here is ignored, and a package run from a source checkout reads + the folder at run time instead.""" here = os.path.dirname(os.path.abspath(__file__)) - source = os.path.join(here, "..", "ip-intelligence-data", "Translations") + source = os.path.join( + here, "..", "..", "cxx", "data", "ip-intelligence", "Translations") dest = os.path.join( here, "src", "fiftyone_ipintelligence_translation", "translations") if not os.path.isdir(source): diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py b/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py index eb9aff55..4c0ffc4c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/src/fiftyone_ipintelligence_translation/resources.py @@ -30,14 +30,15 @@ def _find_submodule_translations(): - """Walk up from this package looking for the ip-intelligence-data - submodule's Translations directory, used when the package is run from a - source checkout rather than an installed distribution.""" + """Walk up from this package looking for the data repository's + Translations directory, cxx/data/ip-intelligence/Translations, used when + the package is run from a source checkout rather than an installed + distribution.""" current = _PACKAGE_DIR for _ in range(8): candidate = os.path.join( - current, "ip-intelligence-data", "Translations") + current, "cxx", "data", "ip-intelligence", "Translations") if os.path.isdir(candidate): return candidate parent = os.path.dirname(current) @@ -50,9 +51,9 @@ def _find_submodule_translations(): def _translation_files(): """Return a dict of translation file name -> absolute path. The packaged - translations directory is preferred; if it is empty the ip-intelligence-data - submodule is used as a fallback (countrycodes at the root, countries under - OSM).""" + translations directory is preferred; if it is empty the data repository's + Translations directory is used as a fallback (countrycodes at the root, + countries under OSM).""" files = {} diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py index 4decac97..199ca3b6 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tests/test_translation.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import os import unittest import yaml @@ -34,6 +35,7 @@ from fiftyone_ipintelligence_translation.countries_translation_engine_builder import ( CountriesTranslationEngineBuilder) from fiftyone_ipintelligence_translation.comparer import make_comparer +from fiftyone_ipintelligence_translation import resources from fiftyone_ipintelligence_translation.resources import ( get_all_countries, get_country_resources) @@ -448,5 +450,18 @@ def test_population_translated_names(self): self.assertNotEqual(geo_codes[0], pop_codes[0]) +class PackagedTranslationsTests(unittest.TestCase): + + def test_translations_are_packaged(self): + # The tests run against the built package. A build that cannot find + # the data repository's Translations folder still succeeds, without + # the YAML files, and the other tests can still pass by reading the + # folder from the checkout, so this checks the package itself. + folder = resources._PACKAGED_TRANSLATIONS + names = os.listdir(folder) if os.path.isdir(folder) else [] + self.assertIn("countrycodes.en_GB.yml", names) + self.assertIn("countries.de_DE.yml", names) + + if __name__ == "__main__": unittest.main() diff --git a/ip-intelligence/ip-intelligence-data b/ip-intelligence/ip-intelligence-data deleted file mode 160000 index b8b3d7d4..00000000 --- a/ip-intelligence/ip-intelligence-data +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b8b3d7d4e7747a0412281b9b48e0144b2ef13fcc diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index 68e79e2e..18d82f7e 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -28,10 +28,9 @@ page shows the Python versions that we currently test against. The on-premise engine wraps the native [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) -library, and the -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) -repository (data and evidence files used for testing) is also included, both -as git submodules. When cloning, use: +library, which is included as the `cxx` git submodule. It has the +[data](https://github.com/51Degrees/data) repository (data and evidence files +used for testing) as its own submodule, at `cxx/data`. When cloning, use: ``` git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git @@ -45,13 +44,25 @@ git submodule update --init --recursive ### Data files -The on-premise engine requires a `.ipi` data file. The -[ip-intelligence-data](https://github.com/51Degrees/ip-intelligence-data) -submodule provides a freely available network-only data file +The on-premise engine requires a `.ipi` data file. The data repository's +`cxx/data/ip-intelligence` folder holds the evidence files used for testing, +and links to a freely available network-only data file (`51Degrees-IPIV4AsnIpiV41.ipi`), which the examples tests and -`CommittedDataFileTests` use, along with the evidence files used for -testing. The on-premise unit tests default to -`assets/51Degrees-LiteIpiV41.ipi`. +`AsnDataFileTests` use. The linked files are not stored in a repository. +Download them from the root of this repository by running: + +``` +sh cxx/data/get-data-files.sh ip-intelligence +``` + +or, in PowerShell: + +``` +./cxx/data/get-data-files.ps1 ip-intelligence +``` + +An example whose file is missing runs the script for that file itself. The +on-premise unit tests default to `assets/51Degrees-LiteIpiV41.ipi`. For the full set of properties (location, registration, etc.) an enterprise data file is required. To obtain one for testing, please @@ -59,12 +70,12 @@ data file is required. To obtain one for testing, please line argument to the examples, or place it within the project space. Note: the engine reads exactly one data file format version, set by -`FIFTYONE_DEGREES_IPI_TARGET_VERSION_MAJOR` and `_MINOR` in the -`ip-intelligence-cxx` submodule's `src/ipi.c`, and refuses any other as -"an unsupported version". Move the `ip-intelligence-cxx` and -`ip-intelligence-data` pins together. `CommittedDataFileTests` in +`FIFTYONE_DEGREES_IPI_TARGET_VERSION_MAJOR` and `_MINOR` in the `cxx` +submodule's `src/ipi/ipi.c`, and refuses any other as "an unsupported +version". The ASN data file is downloaded from the Distributor, which always +serves the latest one published, so `AsnDataFileTests` in [test_ipintelligence.py](fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py) -fails when the two disagree. +fails when the engine cannot read the format 51Degrees publishes. ## Folders @@ -77,8 +88,6 @@ fails when the two disagree. wraps ip-intelligence-cxx via SWIG. - `fiftyone_ipintelligence_shared` - code shared between the engines. - `fiftyone_ipintelligence_examples` - usage examples for both engines. -- `ip-intelligence-data` - submodule containing the test data file and - evidence files used by the examples and tests. ## Installation From 9c11656ae10c68ee96fdeaf169942e4a46f30cad Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:25:12 +0000 Subject: [PATCH 06/26] Say in AGENTS.md how to get the data, build and run the examples (#34) --- AGENTS.md | 258 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 257 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 11855b13..deb8846b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,256 @@ # Working in this repository -Notes on what is not obvious from the code. Keep them short and current. The +If you have been asked to get an example running, follow +[Run the examples](#run-the-examples). It gives every step in order, from a +fresh clone to the output. The sections after it are notes on what is not +obvious from the code, for changing it. Keep them short and current. The layout and how to get the code are in the [README](README.md). +## Run the examples + +Device Detection's on-premise `gettingstarted_console` is the quickest +example to show, because its data file is about 35 MB. IP Intelligence's +on-premise `gettingstarted_console` runs on the free Asn data file, which is +about 6 MB, and shows registration values once it is given the Lite file, +which is a download of about 230 MB. Run each command from the repository root +unless the step says otherwise. + +The on-premise engines are C and C++ code that is compiled when the package is +installed. PyPI has only source distributions of +`fiftyone-devicedetection-onpremise` and `fiftyone-ipintelligence-onpremise`, +so a compiler is needed whether the packages come from PyPI or from this +checkout. The steps below build them from this checkout, as the product +READMEs do. + +### Prerequisites + +- Git. +- Python 3.10 or later, as the README says. CI tests 3.10 to 3.14 on Ubuntu, + Windows and macOS (`ci/options.json`). +- A C11 and C++17 compiler. On Windows that is Visual Studio 2019 or later, or + the [C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/), + with Platform Toolset `v142` and Windows SDK `10.0.18362.0` or later. On + Linux the Device Detection README gives + `sudo apt-get install g++ make libatomic1`. +- For the data files, `curl`, `gzip` and `md5sum` (`md5` on macOS) on Linux + and macOS, where the examples download with the shell script, and Windows + PowerShell 5.1 or PowerShell 7 on Windows, where they use the PowerShell + script. +- Network access to GitHub, to PyPI for the packages the examples depend on, + and to `distributor.51degrees.com` for the data files. + +Clone the repository with its submodules. + +``` +git clone --recurse-submodules https://github.com/51Degrees/51degrees-python.git +cd 51degrees-python +``` + +In a clone made without `--recurse-submodules`, fetch the submodules. + +``` +git submodule update --init --recursive +``` + +### Data files + +The examples use the free data files, which need no licence key. They are not +stored in this repository. The `cxx` submodule holds the +[data](https://github.com/51Degrees/data) repository as `cxx/data`, which has a +link to each file, and a script downloads them into `cxx/data/device-detection` +and `cxx/data/ip-intelligence`. + +On Linux and macOS, run the shell script. + +``` +sh cxx/data/get-data-files.sh +``` + +On Windows, run the PowerShell script. Windows PowerShell can refuse to run a +script file under the policy Windows sets by default, so run it the way the +examples do. + +``` +powershell -NoProfile -ExecutionPolicy Bypass -File cxx/data/get-data-files.ps1 +``` + +Where PowerShell 7 is installed, `pwsh cxx/data/get-data-files.ps1` does the +same on any system. Either script downloads every free file, the Asn file +included, and keeps a file that is already there. To download one file, give +its name, as in `sh cxx/data/get-data-files.sh 51Degrees-LiteV41.ipi`. An +example run with no argument downloads the file it needs itself, when the file +is missing. + +#### Check the files + +- `cxx/data/device-detection/51Degrees-LiteV4.1.hash` is more than 10 MB. +- `cxx/data/ip-intelligence/51Degrees-IPIV4AsnIpiV41.ipi` is more than 1 MB. +- `cxx/data/ip-intelligence/51Degrees-LiteV41.ipi` is more than 100 MB. + +### Build + +Install into a virtual environment, as CI does for the examples +(`device-detection/ci/run-integration-tests.ps1`). On Linux and macOS, make and +activate one with these commands. + +``` +python3 -m venv .venv +. .venv/bin/activate +``` + +On Windows, in PowerShell, run `python -m venv .venv` and then +`.venv\Scripts\Activate.ps1`. + +Then install the Device Detection packages from source, with the commands the +Device Detection README gives, and the examples package. `setuptools` is +added because `python setup.py` needs it and Python 3.12 and later no longer +install it (`ip-intelligence/ci/setup-environment.ps1`). + +``` +cd device-detection +python -m pip install -r requirements.txt setuptools +python -m pip install -e fiftyone_devicedetection_shared/ +python -m pip install -e fiftyone_devicedetection_cloud/ +cd fiftyone_devicedetection_onpremise/ +python setup.py build_clib build_ext +cd ../ +python -m pip install -e fiftyone_devicedetection_onpremise/ +python -m pip install -e fiftyone_devicedetection/ +python -m pip install -e fiftyone_devicedetection_examples/ +cd .. +``` + +Then install the IP Intelligence packages, with the commands the IP +Intelligence README gives. + +``` +cd ip-intelligence +pip install -e fiftyone_ipintelligence_shared +pip install -e fiftyone_ipintelligence_cloud +pip install -e fiftyone_ipintelligence_onpremise +pip install -e fiftyone_ipintelligence +pip install -e fiftyone_ipintelligence_examples +cd .. +``` + +The packages these need from elsewhere, such as the Pipeline API packages, +come from PyPI. CI installs the ones in `pipeline/` over them +(`ci/setup-environment.ps1`), which running an example does not need. + +### Run + +Run the Device Detection example from the repository root with no argument. +It looks for `cxx/data/device-detection/51Degrees-LiteV4.1.hash` in the folder +it runs in and each folder above it, and runs the download script when the +file is missing, with Windows PowerShell on Windows and `sh` on Linux and +macOS (`find_file` in +`device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py`). + +``` +python -m fiftyone_devicedetection_examples.onpremise.gettingstarted_console +``` + +The IP Intelligence example with no argument looks for the Asn file in the +folder it runs in and in its `assets` and `cxx/data/ip-intelligence` folders, +then does the same in each of the seven folders above it, and downloads the +file into `cxx/data/ip-intelligence` when it is missing. To see registration +values, download the Lite file first, as "Data files" says, and give its path +as the first argument. + +``` +python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console +python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +``` + +Either example uses a path given as the first argument as it is, so a file +named that way is never downloaded. + +### Correct output + +When an example downloads its data file, it first prints +` is missing, so it is being downloaded by running` and, on the next +line, the command, then the script's `Downloading` and `Saved` lines. + +The pipeline logger writes through Python's `logging`, so its lines go to +standard error, each as `INFO:root:` or `WARNING:root:` followed by a +dictionary that holds the time, the level and the message. + +The Device Detection example logs `Using a 'Lite' data file created '' +from location ''`, a warning that it is using the Lite file, and possibly +a warning that the file is more than 30 days old. Both warnings are expected. +Then for each of five sets of evidence it prints `Input values:` and the +evidence, then `Results:` with `Mobile Device`, `Platform Name`, +`Platform Version`, `Browser Name` and `Browser Version`. The first set is a +User-Agent from a Samsung Android phone. + +The IP Intelligence example logs `Using a '' data file created +''`, then for each of four IP addresses, starting with `116.154.188.222`, +prints `Input values:` with `query.client-ip`, then `Results:` with a line for +each of 16 properties, starting with `RegisteredName`, `RegisteredOwner` and +`RegisteredCountry`. A property the data file lacks reads +`This property is not present in the data file.` With the Asn file that is +every registration and location property, as the examples' README says. The +Lite file has RegisteredName, RegisteredOwner and RegisteredCountry, so with it +those three lines are read from the file. + +The paid data files have many more properties, as the +[pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=correct-output) +describes. + +### Cloud examples + +The cloud examples need a resource key, which a person creates in the +configurator, so ask for one if none is set. A key with the properties the +Device Detection cloud example uses can be made +[here](https://configure.51degrees.com/1QWJwHxl?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples), +which is the configuration the example's own message names, and a key for IP +Intelligence in the +[configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). + +The Device Detection cloud example takes the key as its first argument, or +from `_51DEGREES_RESOURCE_KEY`, or from the older name `resource_key`, which +the Device Detection README uses. The IP Intelligence cloud example takes it +as its first argument or from `resource_key`. + +``` +resource_key= python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console +resource_key= python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_console +``` + +In PowerShell, set `$env:resource_key = ''` first. Without a key, each +example logs an error that names the variable and the configurator. + +### Pitfalls + +- **A data file that is not there.** `cxx/data` holds a `.link` for each file + rather than the file. An example run with no argument downloads a missing + file, so it needs the network and the tools under "Prerequisites". Check + the file sizes as above. +- **Running outside the repository, or without the submodules.** The Device + Detection example then finds no `cxx/data`, searches for up to 5 seconds + and, when it finds nothing, logs `Failed to find a device detection data + file` with the advice to run `git submodule update --init --recursive`. Run + it from the repository root, in a clone made with Git, because a zip + download of the repository has no submodules. +- **No compiler, or no `setuptools`.** `python setup.py build_clib build_ext` + and the on-premise installs compile C and C++, and on Python 3.12 and later + `python setup.py` stops with `No module named 'setuptools'` until it is + installed. +- **Long paths on Windows.** CI sets `TEMP` to `C:\tmp` on Windows to keep + within the MSVC path length limit + (`device-detection/ci/run-integration-tests.ps1`), so build from a short + folder path. +- **Scripts blocked in Windows PowerShell.** Running `Activate.ps1`, like the + Device Detection README's `setup.ps1`, needs PowerShell to allow scripts, + which that README does with `Set-ExecutionPolicy -Scope CurrentUser Bypass`. +- **Pipenv.** The Device Detection README suggests `pipenv install`, but there + is no `Pipfile` in the repository. Use the virtual environment above. +- **What CI does not test.** CI runs the examples' tests, which call each + example's code rather than its `main`. The Device Detection tests use the + Lite file that `device-detection/ci/fetch-assets.ps1` links in from CI's + assets, and the IP Intelligence example tests use the Asn file and throw + their output away. + ## Products keep their own CI scripts Each product folder still has the `ci/` scripts it had as its own repository, @@ -54,6 +302,14 @@ A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside cxx update therefore releases no package, even when it brings new translations in `cxx/data/ip-intelligence`. +## Releases + +Pull requests go to `rc`, not `main`, and stay drafts until they have been +reviewed, because the nightly merges any pull request into `rc` that is not a +draft and has no review requested. When a release is wanted, one pull request +merges `rc` into `main`, and the "Only rc into main" check fails a pull request +into `main` from any other branch. + ## History The four repositories were brought in with `git subtree add`, so commit SHAs From 7364fe9a84dec2ec6ff31205e034117ee1fb0c23 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:25:36 +0000 Subject: [PATCH 07/26] Point package metadata and package READMEs at the right pages (#35) --- README.md | 2 +- device-detection/ci/README.md | 2 +- .../fiftyone_devicedetection/readme.md | 12 ++++++++++-- .../fiftyone_devicedetection/setup.py | 6 +++++- .../fiftyone_devicedetection_cloud/readme.md | 14 ++++++++++---- .../fiftyone_devicedetection_cloud/setup.py | 6 +++++- .../fiftyone_devicedetection_examples/setup.py | 6 +++++- .../cloud/configurator_console.py | 4 ++-- .../cloud/gettingstarted_console.py | 4 ++-- .../cloud/gettingstarted_web/app.py | 4 ++-- .../cloud/metadata_console.py | 2 +- .../cloud/nativemodellookup_console.py | 4 ++-- .../cloud/taclookup_console.py | 4 ++-- .../cloud/useragentclienthints_web/app.py | 2 +- .../onpremise/gettingstarted_console.py | 6 +++--- .../onpremise/gettingstarted_web/app.py | 2 +- .../onpremise/metadata_console.py | 6 +++--- .../onpremise/offlineprocessing.py | 8 ++++---- .../readme.md | 15 ++++++++++++--- .../fiftyone_devicedetection_onpremise/setup.py | 5 +++++ .../fiftyone_devicedetection_shared/readme.md | 12 ++++++++++-- .../fiftyone_devicedetection_shared/setup.py | 6 +++++- device-detection/readme.md | 6 +++--- ip-intelligence/ci/README.md | 2 +- .../fiftyone_ipintelligence/readme.md | 14 +++++++++++--- .../fiftyone_ipintelligence/setup.py | 6 +++++- .../fiftyone_ipintelligence_cloud/readme.md | 16 ++++++++++++---- .../fiftyone_ipintelligence_cloud/setup.py | 6 +++++- .../fiftyone_ipintelligence_examples/readme.md | 2 +- .../fiftyone_ipintelligence_examples/setup.py | 6 +++++- .../cloud/getallproperties_console.py | 2 +- .../cloud/gettingstarted_console.py | 4 ++-- .../cloud/metadata_console.py | 2 +- .../onpremise/gettingstarted_console.py | 6 +++--- .../fiftyone_ipintelligence_onpremise/readme.md | 15 ++++++++++++--- .../fiftyone_ipintelligence_onpremise/setup.py | 5 +++++ .../fiftyone_ipintelligence_shared/readme.md | 14 +++++++++++--- .../fiftyone_ipintelligence_shared/setup.py | 6 +++++- .../readme.md | 12 ++++++++++-- .../setup.py | 6 +++++- ip-intelligence/readme.md | 4 ++-- location/ci/README.md | 2 +- location/readme.md | 17 ++++++++++++----- location/setup.py | 6 +++++- pipeline/ci/README.md | 2 +- .../readme.md | 14 +++++++++++--- .../setup.py | 6 +++++- .../clouddata.py | 2 +- .../cloudengine.py | 4 ++-- pipeline/fiftyone_pipeline_core/readme.md | 13 ++++++++++--- pipeline/fiftyone_pipeline_core/setup.py | 6 +++++- pipeline/fiftyone_pipeline_did/readme.md | 7 +++++++ pipeline/fiftyone_pipeline_did/setup.py | 6 +++++- pipeline/fiftyone_pipeline_engines/readme.md | 10 ++++++++-- pipeline/fiftyone_pipeline_engines/setup.py | 6 +++++- .../readme.md | 11 +++++++++-- .../fiftyone_pipeline_engines_fiftyone/setup.py | 6 +++++- .../fiftyone_pipeline_translation/readme.md | 11 +++++++++-- pipeline/fiftyone_pipeline_translation/setup.py | 6 +++++- pipeline/readme.md | 4 ++-- 60 files changed, 300 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 2303b4fe..04fb4d05 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ the on-premise engines and the data files. ## Supported Python versions The packages support Python 3.10 and later. The -[tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=tested-versions) +[tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=tested-versions) page lists the versions each release is tested against. ## Versions diff --git a/device-detection/ci/README.md b/device-detection/ci/README.md index 0db0f74d..76a328d6 100644 --- a/device-detection/ci/README.md +++ b/device-detection/ci/README.md @@ -4,7 +4,7 @@ This API complies with the `common-ci` approach. The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests +* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests * Example: `R4nd0m-S7r1ng` * `DEVICE_DETECTION_KEY` - [license key](https://51degrees.com/pricing?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for downloading assets (TAC hashes file and TAC CSV data file) * Example: `V3RYL0NGR4ND0M57R1NG` diff --git a/device-detection/fiftyone_devicedetection/readme.md b/device-detection/fiftyone_devicedetection/readme.md index 7cd26f83..72ffdfb4 100644 --- a/device-detection/fiftyone_devicedetection/readme.md +++ b/device-detection/fiftyone_devicedetection/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Device Detection -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-readme.md&utm_term=51degrees-device-detection-engines-device-detection "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-readme.md&utm_term=51degrees-device-detection-engines-device-detection "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-readme.md&utm_term=51degrees-device-detection-engines-device-detection "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=51degrees-device-detection-engines-device-detection "View all available properties and values") ## Introduction @@ -25,3 +25,11 @@ The Pipeline is a generic web request intelligence and data processing solution To run the tests use: `python -m unittest discover -s tests -p test*.py -b` + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection/setup.py b/device-detection/fiftyone_devicedetection/setup.py index 99b1c73d..e3bd7af0 100644 --- a/device-detection/fiftyone_devicedetection/setup.py +++ b/device-detection/fiftyone_devicedetection/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints, on-premise with a data file or in the cloud. An alternative to UAParser.js, DeviceAtlas, and WURFL."), keywords=[ "51degrees", diff --git a/device-detection/fiftyone_devicedetection_cloud/readme.md b/device-detection/fiftyone_devicedetection_cloud/readme.md index dead810b..10fe2fe4 100644 --- a/device-detection/fiftyone_devicedetection_cloud/readme.md +++ b/device-detection/fiftyone_devicedetection_cloud/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Cloud -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=51degrees-device-detection-engines-cloud "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=51degrees-device-detection-engines-cloud "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=51degrees-device-detection-engines-cloud "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=51degrees-device-detection-engines-cloud "View all available properties and values") ## Introduction @@ -12,7 +12,7 @@ You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/np5M4nlF?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/np5M4nlF?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as exampledd.py * Run the example with `python exampledd.py` * Feel free to try different user-agents and property values. @@ -26,7 +26,7 @@ fd.process() print(fd.device.ismobile.value()) ``` -For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-readme.md&utm_term=from-pypi) page in the documentation. +For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi) page in the documentation. ### From GitHub @@ -40,4 +40,10 @@ To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +## Find out more +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_cloud/setup.py b/device-detection/fiftyone_devicedetection_cloud/setup.py index 0d721623..7318548d 100644 --- a/device-detection/fiftyone_devicedetection_cloud/setup.py +++ b/device-detection/fiftyone_devicedetection_cloud/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints using the 51Degrees cloud service. An alternative to UAParser.js, DeviceAtlas, and WURFL."), keywords=[ "51degrees", diff --git a/device-detection/fiftyone_devicedetection_examples/setup.py b/device-detection/fiftyone_devicedetection_examples/setup.py index 7f6dd0e9..15fa344c 100644 --- a/device-detection/fiftyone_devicedetection_examples/setup.py +++ b/device-detection/fiftyone_devicedetection_examples/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). " "This project contains examples of using 51Degrees Device Detection engines with the Pipeline API"), long_description=read("readme.md"), diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py index b985afe9..9e6d0b5f 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py @@ -31,7 +31,7 @@ # It shows how to call the cloud with the newly created key and how to access the values # of the selected properties. # -# See [Getting Started](https://51degrees.com/documentation/_examples__device_detection__getting_started__console__cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) +# See [Getting Started](https://51degrees.com/documentation/_device_detection__examples__getting_started__console__cloud.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) # for a fuller example. # # Required PyPi Dependencies: @@ -96,7 +96,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index caa7c7b8..3a92915b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -50,7 +50,7 @@ def run(self, config, logger, output): # In this example, we use the PipelineBuilder and configure it from a file. # For more information about builders in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=run # Create the pipeline using the service provider and the configured options. pipeline = PipelineBuilder().add_logger(logger).build_from_configuration(config) @@ -174,7 +174,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " + "service is accessed using a 'ResourceKey'. For more information " + "see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this example can be " + "created for free at https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "Once complete, populate the config file or environment variable " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index ba86890b..8260065b 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -58,7 +58,7 @@ # ``` # # Results can also be accessed in client-side code by using the `fod` object. See the -# [JavaScriptBuilderElementBuilder](https://51degrees.com/pipeline-python/4.3/classpipeline-python_1_1fiftyone__pipeline__core_1_1fiftyone__pipeline__core_1_1javascriptbuilde778a9036818b19ab55d981a40be4a4d7.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=top) +# [JavascriptBuilderElement](https://51degrees.com/documentation/apis/pipeline-python/classfiftyone__pipeline__core_1_1javascriptbuilder_1_1_javascript_builder_element.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=top) # for details on available settings such as changing the `fod` name. # ```{js} # window.onload = function () { @@ -230,7 +230,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/g3gMZdPY?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index 74b0df56..d8b06656 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -120,7 +120,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 4d11f6f5..98b7212e 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -61,7 +61,7 @@ def run(self, resource_key, logger, output, cloudEndPoint = ""): # This example creates the pipeline and engines in code. For a demonstration # of how to do this using a configuration file instead, see the TacLookup example. # For more information about builders in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=run cloudRequestEngineSettings = { "resource_key": resource_key } # If a cloud endpoint has been provided then set the @@ -128,7 +128,7 @@ def main(argv): f"environment variable '{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. " + "The 51Degrees cloud service is accessed using a 'ResourceKey'. " + "For more information " + - "see https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. " + + "see https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. " + "Native model lookup is not available as a free service. This means that " + "you will first need a license key, which can be purchased from our " + "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. Once this is done, a resource " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index 4563bd26..63355916 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -62,7 +62,7 @@ def run(self, config, logger, output): # For a demonstration of how to do this in code instead, see the # NativeModelLookup example. # For more information about builders in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=run # Create the pipeline using the service provider and the configured options. pipeline = PipelineBuilder().add_logger(logger).build_from_configuration(config) @@ -129,7 +129,7 @@ def main(argv): f"the environment variable '{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. " + "The 51Degrees cloud service is accessed using a 'ResourceKey'. " + "For more information see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. " + "TAC lookup is not available as a free service. This means " + "that you will first need a license key, which can be purchased " + "from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. Once this is " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index 730153df..728f0594 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -136,7 +136,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees " + "cloud service is accessed using a 'ResourceKey'. " + "For more detail see " + - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index f49c5bc0..946f06bd 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -51,15 +51,15 @@ def run(self, data_file, logger, output): # In this example, we use the DeviceDetectionPipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run performance_profile = "MaxPerformance", # inhibit sharing usage for this test, usually this # should be set "true" diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index dee9b99b..d382e6f2 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -58,7 +58,7 @@ # ``` # # Results can also be accessed in client-side code by using the `fod` object. See the -# [JavaScriptBuilderElementBuilder](https://51degrees.com/pipeline-python/4.3/classpipeline-python_1_1fiftyone__pipeline__core_1_1fiftyone__pipeline__core_1_1javascriptbuilde778a9036818b19ab55d981a40be4a4d7.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=top) +# [JavascriptBuilderElement](https://51degrees.com/documentation/apis/pipeline-python/classfiftyone__pipeline__core_1_1javascriptbuilder_1_1_javascript_builder_element.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=top) # for details on available settings such as changing the `fod` name. # ```{js} # window.onload = function () { diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index ed974ae6..2df9d104 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -60,15 +60,15 @@ def run(self, data_file, logger, output): # In this example, we use the DeviceDetectionPipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=run performance_profile = "LowMemory", # inhibit sharing usage for this test, usually this # should be set "true" diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 80ca344f..7a90c401 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -73,15 +73,15 @@ def run(self, data_file, evidence_yaml, logger, output): # In this example, we use the DeviceDetectionPipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run performance_profile = "LowMemory", # inhibit sharing usage for this test, usually this # should be set "true" @@ -92,7 +92,7 @@ def run(self, data_file, evidence_yaml, logger, output): # in order to help us improve detection of new devices/browsers/etc, then # this additional data will need to be collected and included as evidence # to the Pipeline. See - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run#Low_Level_Usage_Sharing + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=run#Low_Level_Usage_Sharing # for more details on this. usage_sharing = False, # Inhibit auto-update of the data file for this example diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index 284077ce..5a83973d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - On-Premise -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=51degrees-device-detection-engines-on-premise "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=51degrees-device-detection-engines-on-premise "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=51degrees-device-detection-engines-on-premise "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=51degrees-device-detection-engines-on-premise "View all available properties and values") ## Introduction @@ -56,7 +56,7 @@ fd.process() print(fd.device.ismobile.value()) ``` -For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise-readme.md&utm_term=from-pypi) page in the documentation. +For more in-depth examples, check out the [examples](https://51degrees.com/device-detection-python/examples.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=from-pypi) page in the documentation. ### From GitHub @@ -77,3 +77,12 @@ If you've cloned the GitHub repository, you will be able to run the examples in To run the tests use: `python -m unittest discover -s tests -p test*.py -b` + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python +- https://github.com/51Degrees/51degrees-cxx diff --git a/device-detection/fiftyone_devicedetection_onpremise/setup.py b/device-detection/fiftyone_devicedetection_onpremise/setup.py index d4f60fbc..0af5631d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/setup.py +++ b/device-detection/fiftyone_devicedetection_onpremise/setup.py @@ -232,6 +232,11 @@ def stage_native_source(): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description = """Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints by reading a data file on your server. An alternative to UAParser.js, DeviceAtlas, and WURFL.""", keywords=[ "51degrees", diff --git a/device-detection/fiftyone_devicedetection_shared/readme.md b/device-detection/fiftyone_devicedetection_shared/readme.md index bbff13b6..1a5deb11 100644 --- a/device-detection/fiftyone_devicedetection_shared/readme.md +++ b/device-detection/fiftyone_devicedetection_shared/readme.md @@ -1,8 +1,8 @@ # 51Degrees Device Detection Engines - Shared -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-readme.md&utm_term=51degrees-device-detection-engines-shared "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "THE Fastest and Most Accurate Device Detection") **v4 Device Detection Python** -[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-readme.md&utm_term=51degrees-device-detection-engines-shared "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-readme.md&utm_term=51degrees-device-detection-engines-shared "View all available properties and values") +[Developer Documentation](https://51degrees.com/device-detection-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "Developer Documentation") | [Available Properties](https://51degrees.com/resources/property-dictionary?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=51degrees-device-detection-engines-shared "View all available properties and values") ## Introduction @@ -23,3 +23,11 @@ The Pipeline is a generic web request intelligence and data processing solution To run the tests use: `python -m unittest discover -s tests -p test*.py -b` + +## Find out more + +- [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-device-detection) +- [Device Detection documentation](https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_device_detection__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/device-detection/fiftyone_devicedetection_shared/setup.py b/device-detection/fiftyone_devicedetection_shared/setup.py index e367b7d1..07b89cdf 100644 --- a/device-detection/fiftyone_devicedetection_shared/setup.py +++ b/device-detection/fiftyone_devicedetection_shared/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author_email="engineering@51degrees.com", author="51Degrees Engineering", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). This project contains 51Degrees Device Detection engines that can be used with the Pipeline API."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/device-detection/readme.md b/device-detection/readme.md index 379be178..809d47e4 100644 --- a/device-detection/readme.md +++ b/device-detection/readme.md @@ -25,7 +25,7 @@ git submodule update --init --recursive ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. @@ -35,10 +35,10 @@ The API can either use our cloud service to get its data or it can use a local ( #### Cloud -You will require a [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) +You will require a [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) to use the Cloud API. You can create resource keys using our [configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud), see our -[documentation](https://51degrees.com/documentation/_concepts__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) on how to use this. +[documentation](https://51degrees.com/documentation/_services__configurator.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=cloud) on how to use this. #### On-Premise diff --git a/ip-intelligence/ci/README.md b/ip-intelligence/ci/README.md index a5089246..7f4b5fa1 100644 --- a/ip-intelligence/ci/README.md +++ b/ip-intelligence/ci/README.md @@ -6,7 +6,7 @@ The secrets each workflow passes are named in [.github/workflows](../.github/wor The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `_51DEGREES_RESOURCE_KEY_BESPOKE` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for the cloud tests. It must include the IP Intelligence (`ip`) product, or every cloud test fails as if the service were broken. +* `_51DEGREES_RESOURCE_KEY_BESPOKE` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for the cloud tests. It must include the IP Intelligence (`ip`) product, or every cloud test fails as if the service were broken. * Example: `R4nd0m-S7r1ng` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` diff --git a/ip-intelligence/fiftyone_ipintelligence/readme.md b/ip-intelligence/fiftyone_ipintelligence/readme.md index da4cdcb3..1b1c9eec 100644 --- a/ip-intelligence/fiftyone_ipintelligence/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence/readme.md @@ -1,13 +1,13 @@ # 51Degrees IP Intelligence Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-readme.md&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction This package aggregates the cloud and on-premise 51Degrees IP intelligence -engines for the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-readme.md&utm_term=introduction) +engines for the [51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=introduction) and provides the `IpIntelligencePipelineBuilder` which builds a pipeline with either engine depending on the settings provided: @@ -34,3 +34,11 @@ flowdata.evidence.add("query.client-ip", "185.28.167.78") flowdata.process() print(flowdata.ip.registeredname.value()) ``` + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence/setup.py b/ip-intelligence/fiftyone_ipintelligence/setup.py index ef684aa9..a079441e 100644 --- a/ip-intelligence/fiftyone_ipintelligence/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network, on-premise with a data file or in the cloud. An alternative to MaxMind and IPinfo."), keywords=[ "51degrees", diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md index dba487eb..83d568f9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/readme.md @@ -1,17 +1,17 @@ # 51Degrees IP Intelligence Engines - Cloud -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=51degrees-ip-intelligence-engines-cloud "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=51degrees-ip-intelligence-engines-cloud "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=51degrees-ip-intelligence-engines-cloud "developer documentation") ## Introduction This package uses the 51Degrees cloud service to resolve IP addresses to location, network and registration information using the -[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=introduction). +[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). To use the cloud service you will need a resource key. You can obtain one -from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-readme.md&utm_term=introduction). +from the [51Degrees Configurator](https://configure.51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=introduction). ## Usage @@ -34,3 +34,11 @@ print(flowdata.ip.registeredname.value()) - `tests` - unit tests. These mock the cloud service so no resource key is required; set the `resource_key` environment variable to also run the live cloud integration test. + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py b/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py index 8eca633e..a5cd5e2f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_cloud&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network using the 51Degrees cloud service. An alternative to MaxMind and IPinfo."), keywords=[ "51degrees", diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index d9542f6a..c41c0630 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -2,7 +2,7 @@ ![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=51degrees-ip-intelligence-engines-examples "developer documentation") ## Introduction diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py index 0ae1c723..2e1def33 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Examples for the 51Degrees IP intelligence engines."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py index 0c4eee08..75c8c9d9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/getallproperties_console.py @@ -105,7 +105,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more information " "see " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required. " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-getallproperties_console.py&utm_term=resource-key-required.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py index 882a8b03..7118db82 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/gettingstarted_console.py @@ -81,7 +81,7 @@ def run(self, resource_key, logger, output, cloud_endpoint = None): # In this example, we use the IpIntelligencePipelineBuilder and # configure it in code. For more information about pipelines in # general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run settings = { # Obtain a resource key from https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=run "resource_key": resource_key @@ -172,7 +172,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more " "information see " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " "Once complete, populate the environment variable mentioned at " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py index c4d26b6e..60db3582 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/cloud/metadata_console.py @@ -104,7 +104,7 @@ def main(argv): f"'{ExampleUtils.RESOURCE_KEY_ENV_VAR}'. The 51Degrees cloud " "service is accessed using a 'ResourceKey'. For more information " "see " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required. " "A resource key with the properties required by this example " "can be created for free at https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-cloud-metadata_console.py&utm_term=resource-key-required.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index f5ad8cdc..8e82567b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -75,14 +75,14 @@ def run(self, data_file, logger, output): # In this example, we use the IpIntelligencePipelineBuilder # and configure it in code. For more information about # pipelines in general see the documentation at - # https://51degrees.com/documentation/_concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = IpIntelligencePipelineBuilder( data_file_path = data_file, # We use the low memory profile as its performance is # sufficient for this example. See the documentation for # more detail on this and other configuration options: - # https://51degrees.com/documentation/_features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run - # https://51degrees.com/documentation/_features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run + # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=run performance_profile = "LowMemory", # inhibit sharing usage for this example, usually this # should be set "true" diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 65b11d8e..569058a1 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -1,14 +1,14 @@ # 51Degrees IP Intelligence Engines - OnPremise -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=51degrees-ip-intelligence-engines-onpremise "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=51degrees-ip-intelligence-engines-onpremise "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=51degrees-ip-intelligence-engines-onpremise "developer documentation") ## Introduction This package resolves IP addresses to location, network and registration information using a local `.ipi` data file and the -[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise-readme.md&utm_term=introduction). +[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=introduction). The native [51degrees-cxx](https://github.com/51Degrees/51degrees-cxx/blob/main/docs/ipi/README.md) library is included as a git submodule and wrapped with SWIG. @@ -55,3 +55,12 @@ flowdata.evidence.add("query.client-ip", "185.28.167.78") flowdata.process() print(flowdata.ip.registeredname.value()) ``` + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python +- https://github.com/51Degrees/51degrees-cxx diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py index 6c2d0349..b15a227a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py @@ -250,6 +250,11 @@ def stage_native_source(): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description = """IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network by reading a data file on your server. An alternative to MaxMind and IPinfo.""", keywords=[ "51degrees", diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/readme.md b/ip-intelligence/fiftyone_ipintelligence_shared/readme.md index 7980c868..ced1fe62 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_shared/readme.md @@ -1,14 +1,14 @@ # 51Degrees IP Intelligence Engines - Shared -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-readme.md&utm_term=51degrees-ip-intelligence-engines-shared "Data rewards the curious") **Python IP Intelligence** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-readme.md&utm_term=51degrees-ip-intelligence-engines-shared "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=51degrees-ip-intelligence-engines-shared "developer documentation") ## Introduction This package contains shared code used by both the cloud and on-premise variants of the 51Degrees IP intelligence engines for the -[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-readme.md&utm_term=introduction). +[51Degrees Pipeline API](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=introduction). It is not intended to be used directly. See [fiftyone_ipintelligence](https://pypi.org/project/fiftyone-ipintelligence/) @@ -26,3 +26,11 @@ instead. * The `fiftyone_pipeline_core`, `fiftyone_pipeline_engines`, `fiftyone_pipeline_cloudrequestengine` and `fiftyone_pipeline_engines_fiftyone` packages + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py index c4480d4e..7f89e1d0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Shared code used by the cloud and on-premise variants of the 51Degrees IP Intelligence engines for the 51Degrees Pipeline API."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/readme.md b/ip-intelligence/fiftyone_ipintelligence_translation/readme.md index ad37b2fc..7f12b228 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_translation/readme.md @@ -1,8 +1,8 @@ # IP Intelligence Country Translation -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation-readme.md&utm_term=ip-intelligence-country-translation "Data rewards the curious") **Python IP Intelligence Country Translation** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Data rewards the curious") **Python IP Intelligence Country Translation** -[Developer Documentation](https://51degrees.com/documentation/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation-readme.md&utm_term=ip-intelligence-country-translation "Developer Documentation") +[Developer Documentation](https://51degrees.com/documentation/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=ip-intelligence-country-translation "Developer Documentation") ## Introduction @@ -33,3 +33,11 @@ Built on top of: * [fiftyone_pipeline_core](https://pypi.org/project/fiftyone-pipeline-core/) * [fiftyone_pipeline_translation](https://pypi.org/project/fiftyone-pipeline-translation/) + +## Find out more + +- [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-ip-intelligence) +- [IP Intelligence documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_ip_intelligence__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py index 3f06c4a4..91ca1dd4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_translation/setup.py @@ -80,7 +80,11 @@ def copy_yaml(folder): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_translation&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Country translation engines for 51Degrees on-premise IP Intelligence. Turns the weighted ISO country codes returned by the IP Intelligence engine into localized country names and complete ordered country lists."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/ip-intelligence/readme.md b/ip-intelligence/readme.md index 18d82f7e..7a1be44e 100644 --- a/ip-intelligence/readme.md +++ b/ip-intelligence/readme.md @@ -2,7 +2,7 @@ ![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "Data rewards the curious") **Python IP Intelligence** -[Developer Documentation](https://51degrees.com/ip-intelligence-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") +[Developer Documentation](https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-ip-intelligence-engines "developer documentation") ## Introduction @@ -21,7 +21,7 @@ design of this API. For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) -page. The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) +page. The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. ### Git submodules diff --git a/location/ci/README.md b/location/ci/README.md index 8e1cb35c..399dcd50 100644 --- a/location/ci/README.md +++ b/location/ci/README.md @@ -4,7 +4,7 @@ This API complies with the `common-ci` approach. The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests +* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests * Example: `R4nd0m-S7r1ng` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` diff --git a/location/readme.md b/location/readme.md index 5077f856..e3c7ee9d 100644 --- a/location/readme.md +++ b/location/readme.md @@ -1,9 +1,9 @@ # 51Degrees Geo-Location Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-geo-location-engines) +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines) **v4 Location Python** -[Developer Documentation](https://51degrees.com/location-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=51degrees-geo-location-engines "Developer documentation") +[Developer Documentation](https://51degrees.com/location-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=51degrees-geo-location-engines "Developer documentation") ## Introduction @@ -13,8 +13,8 @@ The Pipeline is a generic web request intelligence and data processing solution ## Dependencies -For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. +For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=dependencies) page. +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ## Installation and Examples @@ -24,7 +24,7 @@ The [tested versions](https://51degrees.com/documentation/_info__tested_versions You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/6CTsmbPx?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/6CTsmbPx?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as examplelocation.py * Run the example with `python examplelocation.py` * Feel free to try different locations and property values. @@ -61,3 +61,10 @@ Execute `export FLASK_APP=web` where `web` is the example file, and start your a Execute `$env:FLASK_APP = "web"` where `web` is the example file, and start your application via `flask run`. +## Find out more + +- [Geolocation](https://51degrees.com/geolocation?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-geolocation) +- [Reverse geocoding documentation](https://51degrees.com/documentation/_reverse_geocoding__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-documentation) +- [Getting started](https://51degrees.com/documentation/_reverse_geocoding__quickstart.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-getting-started) +- [Pricing](https://51degrees.com/pricing?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=find-out-more-pricing) +- https://github.com/51Degrees/51degrees-python diff --git a/location/setup.py b/location/setup.py index 9c8df34b..b7e00ecd 100644 --- a/location/setup.py +++ b/location/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/geolocation?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_reverse_geocoding__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_location&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("51Degrees Geolocation performs reverse geocoding (using longitude and latitude information) to populate postal addresses."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/ci/README.md b/pipeline/ci/README.md index 8e1cb35c..399dcd50 100644 --- a/pipeline/ci/README.md +++ b/pipeline/ci/README.md @@ -4,7 +4,7 @@ This API complies with the `common-ci` approach. The following secrets are required: * `ACCESS_TOKEN` - GitHub [access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#about-personal-access-tokens) for cloning repos, creating PRs, etc. * Example: `github_pat_l0ng_r4nd0m_s7r1ng` -* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_info__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests +* `SUPER_RESOURCE_KEY` - [resource key](https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=ci-readme.md&utm_term=api-specific-ci-cd-approach) for integration tests * Example: `R4nd0m-S7r1ng` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md index 5d82e16d..5979c0e8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/readme.md @@ -1,8 +1,8 @@ # Cloud Request Engine -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-readme.md&utm_term=cloud-request-engine "Data rewards the curious") **Python Pipeline Cloud Request Engine** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Data rewards the curious") **Python Pipeline Cloud Request Engine** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-readme.md&utm_term=cloud-request-engine "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=cloud-request-engine "Developer Documentation") ## Introduction @@ -16,7 +16,7 @@ The Pipeline is a generic web request intelligence and data processing solution This package uses the `engines` class created by the `fiftyone-pipeline-engines`. It makes available: -* A `Cloud Request Engine` which calls the 51Degrees cloud service to fetch properties and metadata about them based on a provided resource key. Get a resource key at https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-readme.md&utm_term=this-package-fiftyone_pipeline_cloudrequestengine +* A `Cloud Request Engine` which calls the 51Degrees cloud service to fetch properties and metadata about them based on a provided resource key. Get a resource key at https://configure.51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=this-package-fiftyone_pipeline_cloudrequestengine * A `Cloud Engine` template which reads data from the Cloud Request Engine. ## Credentials @@ -79,3 +79,11 @@ It is used by the cloud versions of the following 51Degrees engines: - [**fiftyone_devicedetection**](https://pypi.org/project/fiftyone-devicedetection/) - Get details about the devices accessing your web page - [**fiftyone_location**](https://pypi.org/project/fiftyone-location/) - Get postal address details from the location of devices accessing your web page + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-examples) +- [Cloud service documentation](https://51degrees.com/documentation/_services__cloud__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=find-out-more-cloud-service) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py index 434c0a25..223d23d8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py @@ -42,7 +42,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package contains a shared engine that is used to make requests to the 51Degrees cloud service on behalf of other data services such as device detection or location."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py index d28a9c86..4f4f8c88 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/clouddata.py @@ -42,7 +42,7 @@ def check(self, key, flow_element): + flow_element.datakey + " are " + ", ".join(list(flow_element.get_properties().keys())) + ". For more details on resource keys, see our explainer: " - "https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-clouddata.py&utm_term=property-not-found") class CloudData(AspectDataDictionary): diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py index 2d872d16..9f285f75 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/src/fiftyone_pipeline_cloudrequestengine/cloudengine.py @@ -73,7 +73,7 @@ def on_registration(self, pipeline): # key in use, a resource key or a licence key, so a section missing # from them means that key does not carry the product at all. if not self.datakey in pipeline.flow_elements_list["cloud"].flow_element_properties: - raise Exception("The key in use does not include access to any properties under the engine with key " + self.datakey + " that was added to the pipeline. For more details on resource keys, see our explainer: https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=engine-properties-not-accessible " + "Available engine data keys are: " + str([e for e in pipeline.flow_elements_list["cloud"].flow_element_properties])) + raise Exception("The key in use does not include access to any properties under the engine with key " + self.datakey + " that was added to the pipeline. For more details on resource keys, see our explainer: https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=engine-properties-not-accessible " + "Available engine data keys are: " + str([e for e in pipeline.flow_elements_list["cloud"].flow_element_properties])) self.properties = pipeline.flow_elements_list["cloud"].flow_element_properties[self.datakey] @@ -95,7 +95,7 @@ def pipeline_element_not_found(self, element, flowdata): """ - raise Exception("The key in use does not include access to any properties under " + element + ". For more details on resource keys, see our explainer: https://51degrees.com/documentation/_info__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=element-properties-not-accessible " + "Available element data keys are: " + str([e for e in flowdata.pipeline.flow_elements_display_list])) + raise Exception("The key in use does not include access to any properties under " + element + ". For more details on resource keys, see our explainer: https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=comment&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_cloudrequestengine-src-fiftyone_pipeline_cloudrequestengine-cloudengine.py&utm_term=element-properties-not-accessible " + "Available element data keys are: " + str([e for e in flowdata.pipeline.flow_elements_display_list])) def process_internal(self, flowdata): diff --git a/pipeline/fiftyone_pipeline_core/readme.md b/pipeline/fiftyone_pipeline_core/readme.md index 078967db..ad3470f1 100644 --- a/pipeline/fiftyone_pipeline_core/readme.md +++ b/pipeline/fiftyone_pipeline_core/readme.md @@ -1,8 +1,8 @@ # Pipeline Core -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-readme.md&utm_term=pipeline-core "Data rewards the curious") **Python Pipeline Core** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Data rewards the curious") **Python Pipeline Core** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-readme.md&utm_term=pipeline-core "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=pipeline-core "Developer Documentation") ## Introduction @@ -24,4 +24,11 @@ Engines created by 51Degrees: ## Requirements * Python 3.8+ -* The `flask` python library to run the web examples +* The `flask` python library to run the web examples + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_core/setup.py b/pipeline/fiftyone_pipeline_core/setup.py index a79339f4..1da2b014 100644 --- a/pipeline/fiftyone_pipeline_core/setup.py +++ b/pipeline/fiftyone_pipeline_core/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package defines the essential components of the 51Degrees Pipeline API such as flow elements, flow data and evidence. It also packages together JavaScript served by a pipeline and allows for client side requests for additional data populated by evidence from the client side."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index 837353b0..fe51f236 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -713,3 +713,10 @@ is refreshed by common-ci's `update-example-assets` step. not a format limit. - **No creation of new 51Dids.** This is a parser; new 51Dids are issued by the 51Degrees cloud / on-premise hashing engines. + +## Find out more + +- [51Did](https://51degrees.com/developers/51did?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=find-out-more-51did) +- [51Did documentation](https://51degrees.com/documentation/_identifiers_51_did.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=find-out-more-documentation) +- [51Did inspector](https://51degrees.com/developers/51did-inspector?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=find-out-more-51did-inspector) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_did/setup.py b/pipeline/fiftyone_pipeline_did/setup.py index 0c0fb67b..7d5e581b 100644 --- a/pipeline/fiftyone_pipeline_did/setup.py +++ b/pipeline/fiftyone_pipeline_did/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/developers/51did?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_identifiers_51_did.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_did&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("Strongly typed reader and cloud client for the 51Did (51Degrees Identifier) value returned by the 51Degrees Cloud service. Parses the OWID envelope in either base64 alphabet and exposes the Flags, License Id and match key plus the identifier type, and verifies a 51Did's signature offline or through the cloud and redeems a sealed creator context result on the server. Compare match keys, never envelopes."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_engines/readme.md b/pipeline/fiftyone_pipeline_engines/readme.md index 95eebd5d..b9993940 100644 --- a/pipeline/fiftyone_pipeline_engines/readme.md +++ b/pipeline/fiftyone_pipeline_engines/readme.md @@ -1,8 +1,8 @@ # Pipeline Engines -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-readme.md&utm_term=pipeline-engines "Data rewards the curious") **Python Pipeline Engines** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Data rewards the curious") **Python Pipeline Engines** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-readme.md&utm_term=pipeline-engines "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=pipeline-engines "Developer Documentation") ## Introduction @@ -30,3 +30,9 @@ Engines created by 51Degrees: * Python 3.8+ * The gitversion lib using `python -m pip install gitversion` +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_engines/setup.py b/pipeline/fiftyone_pipeline_engines/setup.py index b3b09ee2..d98d758e 100644 --- a/pipeline/fiftyone_pipeline_engines/setup.py +++ b/pipeline/fiftyone_pipeline_engines/setup.py @@ -41,7 +41,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package extends the flow element class created by the fiftyone-pipeline-core package into a specialized type of flow element called an engine."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md b/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md index 49d04854..8dba8ae2 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/readme.md @@ -1,8 +1,8 @@ # Pipeline Engines Fiftyone -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-readme.md&utm_term=pipeline-engines-fiftyone "Data rewards the curious") **Python Pipeline Engines** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Data rewards the curious") **Python Pipeline Engines** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-readme.md&utm_term=pipeline-engines-fiftyone "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=pipeline-engines-fiftyone "Developer Documentation") ## Introduction @@ -15,3 +15,10 @@ It includes a ShareUsage engine that sends usage data to 51Degrees in zipped bat ## Requirements * Python 3.8+ + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py b/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py index a4c89ef1..b9445460 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/setup.py @@ -40,7 +40,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_engines_fiftyone&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). It includes a ShareUsage engine that sends usage data to 51Degrees in zipped batches."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/fiftyone_pipeline_translation/readme.md b/pipeline/fiftyone_pipeline_translation/readme.md index e38c02dd..4c0450eb 100644 --- a/pipeline/fiftyone_pipeline_translation/readme.md +++ b/pipeline/fiftyone_pipeline_translation/readme.md @@ -1,8 +1,8 @@ # Pipeline Translation -![51Degrees](https://51degrees.com/img/logo.png?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation-readme.md&utm_term=pipeline-translation "Data rewards the curious") **Python Pipeline Translation** +![51Degrees](https://51degrees.com/img/logo.png?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Data rewards the curious") **Python Pipeline Translation** -[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation-readme.md&utm_term=pipeline-translation "Developer Documentation") +[Developer Documentation](https://51degrees.com/pipeline-python/index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=pipeline-translation "Developer Documentation") ## Introduction @@ -24,3 +24,10 @@ header). This package is built on top of: * [fiftyone_pipeline_core](https://pypi.org/project/fiftyone-pipeline-core/) + +## Find out more + +- [Build your own](https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-build-your-own) +- [Pipeline API documentation](https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-documentation) +- [Pipeline API examples](https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=find-out-more-examples) +- https://github.com/51Degrees/51degrees-python diff --git a/pipeline/fiftyone_pipeline_translation/setup.py b/pipeline/fiftyone_pipeline_translation/setup.py index fa2a9f4d..6460ddb6 100644 --- a/pipeline/fiftyone_pipeline_translation/setup.py +++ b/pipeline/fiftyone_pipeline_translation/setup.py @@ -43,7 +43,11 @@ def read(file_name): version=read("version.txt"), author="51Degrees Engineering", author_email="engineering@51degrees.com", - url="https://51degrees.com/?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation-setup.py&utm_term=url", + project_urls={ + "Homepage": "https://51degrees.com/build-your-own?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=homepage", + "Documentation": "https://51degrees.com/documentation/_pipeline_api__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_translation&utm_term=documentation", + "Source": "https://github.com/51Degrees/51degrees-python", + }, description=("This package adds a generic translation flow element to the fiftyone-pipeline, which translates values from a source element into another language using YAML translation files."), long_description=read("readme.md"), long_description_content_type='text/markdown', diff --git a/pipeline/readme.md b/pipeline/readme.md index a5c12c34..ea7419ec 100755 --- a/pipeline/readme.md +++ b/pipeline/readme.md @@ -24,7 +24,7 @@ The examples are described under Examples below. The `owid-python` folder is a g ## Dependencies For runtime dependencies, see our [dependencies](https://51degrees.com/documentation/_info__dependencies.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page. -The [tested versions](https://51degrees.com/documentation/_info__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. +The [tested versions](https://51degrees.com/documentation/_product_summaries__tested_versions.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=dependencies) page shows the Python versions that we currently test against. The software may run fine against other versions, but additional caution should be applied. ## Installation @@ -56,7 +56,7 @@ The tests that call the 51Degrees cloud service read a resource key from the `re ## Examples There are several examples available that demonstrate how to make use of the Pipeline API in isolation. These are described in the table below. -If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [folder](https://github.com/51Degrees/51degrees-python/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_examples__device_detection__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). +If you want examples that demonstrate how to use 51Degrees products such as device detection, then these are available in the corresponding [folder](https://github.com/51Degrees/51degrees-python/tree/main/device-detection) and on our [website](https://51degrees.com/documentation/_device_detection__examples__index.html?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=examples). | Example | Description | |-----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| From 47cc252b8e26678f547e725983ee8a8d9d2c9b97 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:25:57 +0000 Subject: [PATCH 08/26] Ask the country property for its no value message in the location example (#32) --- location/examples/cloud/gettingstarted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/location/examples/cloud/gettingstarted.py b/location/examples/cloud/gettingstarted.py index f5a4041c..a0052c3f 100644 --- a/location/examples/cloud/gettingstarted.py +++ b/location/examples/cloud/gettingstarted.py @@ -77,4 +77,4 @@ if flow_data.location.country.has_value(): print(flow_data.location.country.value()) else: - print(flow_data.location.no_value_message()) + print(flow_data.location.country.no_value_message()) From 653e5521aed4c311fe463487cf7f97fddbbf018a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 17:26:19 +0000 Subject: [PATCH 09/26] Describe the rc approach to releases in AGENTS.md (#29) --- AGENTS.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index deb8846b..539b95af 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -310,6 +310,14 @@ draft and has no review requested. When a release is wanted, one pull request merges `rc` into `main`, and the "Only rc into main" check fails a pull request into `main` from any other branch. +## Releases + +Pull requests go to `rc`, not `main`, and stay drafts until they have been +reviewed, because the nightly merges any pull request into `rc` that is not a +draft and has no review requested. When a release is wanted, one pull request +merges `rc` into `main`, and the "Only rc into main" check fails a pull request +into `main` from any other branch. + ## History The four repositories were brought in with `git subtree add`, so commit SHAs From b3e0cd757870808c17fdb52833544a1aead9aab4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 19:53:17 +0000 Subject: [PATCH 10/26] Check the callback URL before the JavaScript builder writes it (#31) --- .../fiftyone_pipeline_core/callback_url.py | 274 ++++++++++++++++++ .../javascriptbuilder.py | 44 +-- .../tests/test_callback_url.py | 200 +++++++++++++ .../tests/test_javascriptbuilder_variables.py | 84 ++++++ 4 files changed, 581 insertions(+), 21 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_callback_url.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py new file mode 100644 index 00000000..afafa240 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -0,0 +1,274 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The callback URL the client script posts to for updated JSON. The protocol +and host usually come from the request, and the script holds the URL in a +single-quoted string, so the URL is checked before it is written into the +script. A URL that is not valid is refused. In the user information, in +anything after the host and port, and in the endpoint, the characters a URL +cannot hold or that could end the string or the script are percent-encoded. + +See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url +""" + +import re +from collections import namedtuple + +# Whether the URL can be written into the script, and the URL. That is the +# URL to write into the script when it is valid, and otherwise the URL as it +# was built, for the log. +CallbackUrl = namedtuple("CallbackUrl", ["valid", "url"]) + +# The largest port number. +MAX_PORT = 65535 + +# The most digits a port can have once its leading zeros are dropped. +MAX_PORT_DIGITS = 5 + +# The printable ASCII characters that are percent-encoded, besides the +# control characters, the space and everything outside ASCII. +ENCODED_CHARACTERS = frozenset(b"\"'<>\\^`{|}") + +HEX_DIGITS = frozenset(b"0123456789ABCDEFabcdef") + +# Text made only of the Unicode White_Space characters, which counts as a +# part that is not set. The first range is tab, line feed, vertical tab, +# form feed and carriage return. +BLANK_PATTERN = re.compile( + r"[\t-\r \u0085\u00a0\u1680\u2000-\u200a" + r"\u2028\u2029\u202f\u205f\u3000]*") + +# A scheme as RFC 3986 section 3.1 defines one, being a letter followed by +# letters, digits, '+', '-' and '.'. +SCHEME_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") + +# Where the authority ends, which is at the first '/', '?' or '#' of the +# host. +AUTHORITY_END_PATTERN = re.compile(r"[/?#]") + +# A label of a host name, being ASCII letters, digits, '-' and '_'. +LABEL_PATTERN = re.compile(r"[A-Za-z0-9_-]+") + +# An IPv6 zone identifier, being unreserved characters as RFC 3986 section +# 2.3 defines them. +ZONE_PATTERN = re.compile(r"[A-Za-z0-9._~-]+") + +# A port as it follows the host, being ':' and any number of digits. +PORT_PATTERN = re.compile(r":[0-9]*") + +# One group of an IPv6 address. +HEX_GROUP_PATTERN = re.compile(r"[0-9A-Fa-f]{1,4}") + +# An IPv4 address as RFC 3986 section 3.2.2 defines one, being four numbers +# from 0 to 255 with no leading zeros. +_DEC_OCTET = r"(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])" +IPV4_PATTERN = re.compile(_DEC_OCTET + r"(\." + _DEC_OCTET + r"){3}") + +# A lone surrogate, which has no UTF-8 encoding, so it is written as U+FFFD, +# as a browser writes it. +SURROGATE_PATTERN = re.compile(r"[\ud800-\udfff]") + + +def is_blank(text): + + """! + Whether a part of the URL is not set. + + @type text: object + @param text: The part + @rtype: bool + @return: True if the part is not a string, or is empty or only + whitespace + """ + + return (not isinstance(text, str) + or BLANK_PATTERN.fullmatch(text) is not None) + + +def callback_url(protocol, host, endpoint): + + """! + Build the callback URL from its parts and check it. One slash separates + the host and the endpoint, being added when neither has one and removed + when both do. The protocol and the host are written in lower case and + the rest as given, apart from the percent-encoding. + + @type protocol: str + @param protocol: The protocol, such as https + @type host: str + @param host: The host, which can start with user information and end + with a port, a path, a query or a fragment + @type endpoint: str + @param endpoint: The endpoint + @rtype: CallbackUrl + @return: The callback URL, or None when the protocol, host or endpoint + is not set + """ + + if is_blank(protocol) or is_blank(host) or is_blank(endpoint): + return None + host_has_slash = host.endswith("/") + endpoint_has_slash = endpoint.startswith("/") + if not host_has_slash and not endpoint_has_slash: + endpoint = "/" + endpoint + elif host_has_slash and endpoint_has_slash: + endpoint = endpoint[1:] + built = protocol + "://" + host + endpoint + if SCHEME_PATTERN.fullmatch(protocol) is None: + return CallbackUrl(False, built) + match = AUTHORITY_END_PATTERN.search(host) + end = len(host) if match is None else match.start() + authority = _checked_authority(host[:end]) + if authority is None: + return CallbackUrl(False, built) + return CallbackUrl( + True, + protocol.lower() + "://" + authority + + _encoded(host[end:]) + _encoded(endpoint)) + + +def _checked_authority(authority): + + """! + The user information, host and port as they are written into the URL, + or None when they are not valid. The user information ends at the first + '@', so a second '@' is part of the host and refused. + """ + + user, at, host_port = authority.partition("@") + if not at: + return _checked_host_port(authority) + host_port = _checked_host_port(host_port) + if host_port is None: + return None + return _encoded(user) + "@" + host_port + + +def _checked_host_port(host_port): + + """! + The host and optional port in lower case, or None when they are not + valid. The host is an IPv6 address in brackets with an optional zone + identifier, or a name of labels separated by dots with a dot allowed at + the end. The port is ':' followed by digits up to 65535, or by nothing. + """ + + if host_port.startswith("["): + close = host_port.find("]") + if close == -1: + return None + address, separator, zone = host_port[1:close].partition("%25") + if separator and ZONE_PATTERN.fullmatch(zone) is None: + return None + if not _is_ipv6(address): + return None + host = host_port[:close + 1] + port = host_port[close + 1:] + else: + host, colon, port = host_port.partition(":") + port = colon + port + name = host[:-1] if host.endswith(".") else host + for label in name.split("."): + if LABEL_PATTERN.fullmatch(label) is None: + return None + if port and not _is_port(port): + return None + return host.lower() + port + + +def _is_port(text): + + """! + Whether the text is ':' followed by digits whose value is no more than + 65535, or by nothing. + """ + + if PORT_PATTERN.fullmatch(text) is None: + return False + digits = text[1:].lstrip("0") + return len(digits) <= MAX_PORT_DIGITS and ( + not digits or int(digits) <= MAX_PORT) + + +def _is_ipv6(text): + + """! + Whether the text is an IPv6 address as RFC 3986 section 3.2.2 defines + one. That is eight groups of one to four hexadecimal digits separated by + ':', where one '::' can stand for one or more groups of zeros and the + last two groups can be written as an IPv4 address. + """ + + gap = text.find("::") + if gap == -1: + return _groups(text, True) == 8 + if text.find("::", gap + 1) != -1: + return False + head = _groups(text[:gap], False) + tail = _groups(text[gap + 2:], True) + return head >= 0 and tail >= 0 and head + tail <= 7 + + +def _groups(run, ipv4_at_end): + + """! + The number of 16 bit groups in a run of groups separated by ':', or -1 + when the run is not valid. An empty run has none, and an IPv4 address at + the end of the run, where one is allowed, counts as two. + """ + + if not run: + return 0 + parts = run.split(":") + count = 0 + for i, part in enumerate(parts): + if HEX_GROUP_PATTERN.fullmatch(part): + count += 1 + elif (ipv4_at_end and i == len(parts) - 1 + and IPV4_PATTERN.fullmatch(part)): + count += 2 + else: + return -1 + return count + + +def _encoded(text): + + """! + The text with each byte of its UTF-8 encoding percent-encoded where the + byte is a control character, a space, outside ASCII or one of + ENCODED_CHARACTERS, and with a '%' encoded where two hexadecimal digits + do not follow it. + """ + + data = SURROGATE_PATTERN.sub("\ufffd", text).encode("utf-8") + out = [] + for i, b in enumerate(data): + if b == 0x25: + keep = (i + 2 < len(data) + and data[i + 1] in HEX_DIGITS + and data[i + 2] in HEX_DIGITS) + else: + keep = 0x20 < b < 0x7F and b not in ENCODED_CHARACTERS + out.append(chr(b) if keep else "%{:02X}".format(b)) + return "".join(out) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 1f9e206b..31d063a7 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -38,6 +38,7 @@ from .evidence_keyfilter import EvidenceKeyFilter from .elementdata_dictionary import ElementDataDictionary from .constants import Constants +from .callback_url import callback_url # Evidence keys the rendered script is not configured with. Both are @@ -295,6 +296,10 @@ def __init__(self, settings = {} ): self.exclude_from_messages = True + # Set once a callback URL that is not valid has been logged, because + # a misconfigured element would otherwise log on every request. + self._invalid_url_logged = False + # Load template file contents into memory template = Path(__file__).absolute().parent / "js_templates" / "JavaScriptResource.mustache" @@ -413,27 +418,24 @@ def process_internal(self, flowdata): variables["_parameters"] = json.dumps( variables["_parameters"], separators=(",", ":")) - if variables["_host"] and variables["_protocol"] and variables["_endpoint"]: - - # The callback URL is the protocol, the host and the endpoint, - # with exactly one slash between the host and the endpoint, as - # in the .NET builder. No query values are added to it. They - # reach the callback in the request body through the parameters - # above, and the script appends the session id and the sequence - # itself. Adding every evidence value here, as this builder used - # to, sent those two twice and, because the evidence reaching - # this element is not narrowed to query values, also wrote the - # end user's User-Agent, IP address and host headers into the - # served script's URL. - host = variables["_host"] - endpoint = variables["_endpoint"] - if not endpoint.startswith("/") and not host.endswith("/"): - endpoint = "/" + endpoint - elif endpoint.startswith("/") and host.endswith("/"): - endpoint = endpoint[1:] - - variables["_url"] = variables["_protocol"] + "://" + host + endpoint - + # The callback URL is the protocol, the host and the endpoint, as in + # the .NET builder, and callback_url checks it. No query values are + # added to it, because they reach the callback in the request body + # through the parameters above, and the script appends the session + # id and the sequence itself. + url = callback_url( + variables["_protocol"], variables["_host"], variables["_endpoint"]) + if url is not None and not url.valid and not self._invalid_url_logged: + self._invalid_url_logged = True + self._log( + "error", + "The generated URL " + repr(url.url) + " is not valid, so " + "the JavaScript has no callback URL. Check the JavaScript " + "builder's protocol, host and endpoint, and the protocol and " + "host evidence. This is reported once.") + + if url is not None and url.valid: + variables["_url"] = url.url variables["_updateEnabled"] = True else: variables["_updateEnabled"] = False diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py new file mode 100644 index 00000000..9048401d --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -0,0 +1,200 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Checks the callback URL against the cases every implementation of the +JavaScript builder shares, in the same order, so that each one refuses and +writes the same URLs. +""" + +import unittest + +from parameterized import parameterized + +from fiftyone_pipeline_core.callback_url import CallbackUrl, callback_url + +ENDPOINT = "/51dpipeline/json" + + +def _url(protocol, host): + return callback_url(protocol, host, ENDPOINT) + + +def _valid(text): + return CallbackUrl(True, text) + + +class CallbackUrlTests(unittest.TestCase): + + def test_adds_a_single_slash(self): + self.assertEqual( + _valid("https://example.com/51dpipeline/json"), + callback_url("https", "example.com", "51dpipeline/json")) + + def test_keeps_a_single_slash_on_the_endpoint(self): + self.assertEqual( + _valid("https://example.com/51dpipeline/json"), + callback_url("https", "example.com", "/51dpipeline/json")) + + def test_collapses_a_double_slash(self): + self.assertEqual( + _valid("http://example.com/json"), + callback_url("http", "example.com/", "/json")) + + def test_missing_when_a_part_is_empty(self): + self.assertIsNone(callback_url("https", "", "/json")) + self.assertIsNone(callback_url("", "example.com", "/json")) + self.assertIsNone(callback_url("https", "example.com", "")) + self.assertIsNone(callback_url("https", " ", "/json")) + + @parameterized.expand([ + ("EXAMPLE.com", "https://example.com/51dpipeline/json"), + ("example.com:8080", "https://example.com:8080/51dpipeline/json"), + ("example.com:", "https://example.com:/51dpipeline/json"), + ("example.com:65535", "https://example.com:65535/51dpipeline/json"), + ("a_b.com", "https://a_b.com/51dpipeline/json"), + ("-ab.com", "https://-ab.com/51dpipeline/json"), + ("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + ("[::1]", "https://[::1]/51dpipeline/json"), + ("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), + ("[fe80::1%25eth0]", "https://[fe80::1%25eth0]/51dpipeline/json"), + ("user:pw@example.com", + "https://user:pw@example.com/51dpipeline/json"), + ("u x@example.com", "https://u%20x@example.com/51dpipeline/json"), + ("u\tx@example.com", "https://u%09x@example.com/51dpipeline/json"), + ("example.com.", "https://example.com./51dpipeline/json"), + ("example.com/path", "https://example.com/path/51dpipeline/json"), + ("example.com?x", "https://example.com?x/51dpipeline/json"), + ("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), + ("example.com/a%zzb", + "https://example.com/a%25zzb/51dpipeline/json"), + ("example.com/a%41b", "https://example.com/a%41b/51dpipeline/json"), + ("example.com/\u00fc", + "https://example.com/%C3%BC/51dpipeline/json"), + ("example.com/a\nb", "https://example.com/a%0Ab/51dpipeline/json"), + ("example.com/a\u2028b", + "https://example.com/a%E2%80%A8b/51dpipeline/json"), + ]) + def test_accepts_the_host(self, host, expected): + self.assertEqual(_valid(expected), _url("https", host)) + + @parameterized.expand([ + ("example.com:65536",), + ("example.com:99999",), + ("example.com:abc",), + ("exa mple.com",), + ("example.com');alert(1);//",), + ("a\"b.com",), + ("a'b.com",), + ("a",), + ]) + def test_nothing_given_can_end_the_string(self, host): + """A quote, a backslash or a line break never reaches the script, + wherever in the URL it was given.""" + + result = _url("https", host) + self.assertTrue(result.valid, repr(host)) + for c in "'\"\\<>\n\r\u2028\u2029": + self.assertNotIn(c, result.url, repr(host)) + + def test_the_endpoint_is_encoded(self): + self.assertEqual( + _valid("https://example.com/json%27x"), + callback_url("https", "example.com", "/json'x")) + + @parameterized.expand([ + ("b\u00fccher.de",), + ("a\u00a0b.com",), + ("a\u200bb.com",), + ("a\u2028b.com",), + ]) + def test_refuses_the_host_outside_ascii(self, host): + """A host must be ASCII, so an internationalized name is given as + A-labels, and anything else outside ASCII is refused.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + def test_writes_a_lone_surrogate_as_the_replacement_character(self): + """Not one of the shared cases, because only some languages' strings + can hold a lone surrogate. It has no UTF-8 encoding, so it is written + as U+FFFD, as a browser writes it.""" + + self.assertEqual( + _valid("https://example.com/%EF%BF%BD/51dpipeline/json"), + _url("https", "example.com/\ud800")) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index d2c13a41..4c9302f4 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -27,6 +27,40 @@ from fiftyone_pipeline_core import javascriptbuilder from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from .classes.memorylogger import MemoryLogger + + +class _CapturedAll: + """Records the variables of every render and renders nothing.""" + + def __init__(self): + self.variables = [] + + def render(self, template, variables): + self.variables.append(dict(variables)) + return "" + + +def _render_with_hosts(hosts): + """Renders once for each host given, all through the same element, with + the host taken from the request's evidence. Returns the variables of + each render and the errors the pipeline logged.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json"}}).add_logger(logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + for host in hosts: + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.host", host) + flowdata.process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + return captured.variables, errors + class _Captured: """Records the variables of the last render and renders nothing.""" @@ -147,6 +181,56 @@ def test_url_keeps_a_query_string_given_in_the_endpoint(self): self.assertEqual( "https://example.com/json?tenant=1", variables["_url"]) + def test_refused_url_is_left_out_and_logged_once(self): + """A host the check refuses gives a script with no callback, and the + refusal is logged as an error the first time only, whatever the + requests in between.""" + + rendered, errors = _render_with_hosts( + ["exa mple.com", "exa mple.com", "example.com", "exa mple.com"]) + + for i, variables in enumerate(rendered): + if i == 2: + self.assertEqual( + "https://example.com/json", variables["_url"]) + self.assertTrue(variables["_updateEnabled"]) + else: + self.assertNotIn("_url", variables) + self.assertFalse(variables["_updateEnabled"]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'https://exa mple.com/json'", errors[0]) + + def test_refused_url_is_logged_on_one_line(self): + """The URL is quoted and escaped in the log, so a line break in it + cannot start a new log line.""" + + _, errors = _render_with_hosts(["a\nb.com\r\u2028"]) + + self.assertEqual(1, len(errors), errors) + for c in "\n\r\u2028": + self.assertNotIn(c, errors[0]) + self.assertIn("a\\nb.com\\r\\u2028", errors[0]) + + def test_blank_host_gives_no_callback_and_is_not_logged(self): + """A host made only of whitespace is not set, so the script has no + callback and nothing is logged.""" + + rendered, errors = _render_with_hosts([" "]) + + self.assertNotIn("_url", rendered[0]) + self.assertFalse(rendered[0]["_updateEnabled"]) + self.assertEqual([], errors) + + def test_host_from_the_request_is_written_as_checked(self): + """A host from the request reaches the script as the check writes + it, in lower case and with anything that could end the string the + script holds it in percent-encoded.""" + + rendered, _ = _render_with_hosts(["EXAMPLE.com/');alert(1);//"]) + + self.assertEqual( + "https://example.com/%27);alert(1);//json", rendered[0]["_url"]) + if __name__ == "__main__": unittest.main() From 85c7a5cd363f85c3669d713f4818afa2b7c01adf Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 21:00:37 +0000 Subject: [PATCH 11/26] Let FlowData and pipelines release native memory, and close them in the tests and examples (#36) --- .gitignore | 4 + AGENTS.md | 9 + ci/setup-environment.ps1 | 4 + ci/write-package-versions.ps1 | 28 +++ device-detection/ci/run-integration-tests.ps1 | 7 +- .../examples/gettingstarted.py | 61 +++--- .../performance-tests/test_performance.py | 31 +-- .../setup.py | 4 +- .../onpremise/datafileupdate_console.py | 92 ++++----- .../onpremise/failuretomatch.py | 61 +++--- .../onpremise/gettingstarted_console.py | 82 ++++---- .../onpremise/gettingstarted_web/app.py | 95 ++++----- .../onpremise/match_metrics.py | 148 +++++++------- .../onpremise/metadata_console.py | 10 +- .../onpremise/offlineprocessing.py | 141 +++++++------- .../onpremise/performance.py | 65 ++++--- .../onpremise/shareusage.py | 85 ++++---- .../onpremise/useragentclienthints_web/app.py | 49 ++--- .../tests/conftest.py | 119 ++++++++++++ .../tests/test_onpremisegettingstartedweb.py | 7 + .../fiftyone_devicedetection_examples/tox.ini | 6 + .../setup.py | 9 +- .../constants.py | 7 + .../devicedetection_onpremise.py | 17 ++ .../swig_data.py | 29 +++ .../tests/conftest.py | 119 ++++++++++++ .../tests/test_automatic_data_file_updates.py | 47 +++-- .../tests/test_devicedetection.py | 132 ++++++++++++- .../tests/test_properties.py | 7 + .../tests/test_swig.py | 4 + .../tox.ini | 6 + .../examples/gettingstarted.py | 59 +++--- .../fiftyone_ipintelligence_examples/setup.py | 2 + .../onpremise/compare_console.py | 117 +++++------ .../onpremise/countries_console.py | 69 ++++--- .../onpremise/datafileupdate_console.py | 62 +++--- .../onpremise/gettingstarted_console.py | 64 +++--- .../onpremise/gettingstarted_web/app.py | 84 ++++---- .../onpremise/metadata_console.py | 17 +- .../onpremise/mixed_console.py | 51 ++--- .../onpremise/offlineprocessing.py | 60 +++--- .../onpremise/performance_console.py | 25 ++- .../tests/conftest.py | 119 ++++++++++++ .../tests/test_examples.py | 2 + .../fiftyone_ipintelligence_examples/tox.ini | 7 + .../setup.py | 9 +- .../constants.py | 7 + .../ipintelligence_onpremise.py | 17 ++ .../swig_data.py | 29 +++ .../tests/conftest.py | 119 ++++++++++++ .../tests/test_ipintelligence.py | 53 +++++ .../fiftyone_ipintelligence_onpremise/tox.ini | 6 + .../src/fiftyone_pipeline_core/flowdata.py | 28 +++ .../src/fiftyone_pipeline_core/flowelement.py | 11 ++ .../src/fiftyone_pipeline_core/pipeline.py | 48 ++++- .../fiftyone_pipeline_core/pipelinebuilder.py | 63 ++++-- .../tests/test_close.py | 182 ++++++++++++++++++ .../datafile_update_service.py | 48 ++++- .../src/fiftyone_pipeline_engines/engine.py | 16 ++ .../tests/test_datafile_update_service.py | 39 ++++ .../share_usage.py | 11 ++ 61 files changed, 2148 insertions(+), 761 deletions(-) create mode 100644 ci/write-package-versions.ps1 create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/conftest.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_core/tests/test_close.py diff --git a/.gitignore b/.gitignore index 7c20e4e1..21cc57bb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ __pycache__/ .venv/ build/ dist/ + +# The version each package's setup.py reads, written by the publish and by +# ci/write-package-versions.ps1. +version.txt diff --git a/AGENTS.md b/AGENTS.md index 539b95af..81f3a40b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,6 +285,15 @@ installed the pipeline packages from PyPI. In CI the root `ci/setup-environment.ps1` installs them from `pipeline/` over those, so a pipeline change is tested with every product in the same run. +A package's `setup.py` reads its version from a `version.txt`, which only +`pipeline/fiftyone_pipeline_did` commits, so any other package installed from +a checkout is 0.0.0. That is below the minimum versions of the pipeline +packages that the on-premise and examples packages require, so +`ci/write-package-versions.ps1` gives each published package the version this +commit would release it at. The root `ci/setup-environment.ps1` runs it +first. Run `pwsh ci/write-package-versions.ps1` before running those +packages' tests from a checkout. + ## Submodules The C/C++ engines are the `cxx` submodule, listed in the root `.gitmodules`. diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index 9994e2ef..932f5d81 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -12,6 +12,10 @@ if ($env:GITHUB_JOB -eq "PreBuild") { exit 0 } +# Before anything installs a package from this checkout, so that each one +# installs at the version this commit would release it at. +& (Join-Path $PSScriptRoot 'write-package-versions.ps1') -RepoName $RepoName + Invoke-ProductStep 'setup-environment' $RepoName $PSBoundParameters # The device detection, IP intelligence and location scripts install the diff --git a/ci/write-package-versions.ps1 b/ci/write-package-versions.ps1 new file mode 100644 index 00000000..44b23f25 --- /dev/null +++ b/ci/write-package-versions.ps1 @@ -0,0 +1,28 @@ +param ( + [string]$RepoName = '.' +) +$ErrorActionPreference = "Stop" +. (Join-Path $PSScriptRoot 'packages.ps1') + +# Writes the version.txt that each published package's setup.py reads. Without +# one a package installed from this checkout is 0.0.0, which is below the +# versions the packages here require of each other, so pip refuses it, or +# replaces it with a release from PyPI. Each package gets the version this +# commit would release it at, as the publish gives it when it builds the +# package, or its last release when this commit would not release it. +$plan = @(Get-ReleasePlan $RepoName) +$published = @(Get-Packages $RepoName | Where-Object { $_.Publish -eq $true }) +foreach ($package in $published) { + # A version.txt committed with the package is left as it is. + if (git -C $RepoName ls-files -- "$($package.Path)/version.txt") { + continue + } + $release = $plan | Where-Object { $_.Name -eq $package.Name } + if ($null -ne $release) { + $version = $release.Version + } else { + $version = "$((Get-ReleasedVersions $RepoName $package.Name)[-1])" + } + $version | Out-File (Join-Path $RepoName $package.Path 'version.txt') + Write-Output "$($package.Name) $version" +} diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index 9aa2cf2f..f1483528 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -67,7 +67,12 @@ Push-Location $examplesDir try { python3 -m venv .virtualenv if ($LASTEXITCODE -ne 0) { throw "failed to create the examples virtual environment" } - & $py -m pip install -e . + # The pipeline packages come from this checkout rather than a package + # index, so the examples run with the pipeline code they are released with. + $pipeline = 'fiftyone_pipeline_core', 'fiftyone_pipeline_engines', + 'fiftyone_pipeline_engines_fiftyone', 'fiftyone_pipeline_cloudrequestengine' + $local = $pipeline | ForEach-Object { '-e'; "../../pipeline/$_" } + & $py -m pip install -e . @local if ($LASTEXITCODE -ne 0) { throw "failed to install the examples package" } } finally { Pop-Location } diff --git a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py index cf3a7b7f..b439c3d3 100644 --- a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py +++ b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py @@ -35,46 +35,49 @@ # Cloud #pipeline = DeviceDetectionPipelineBuilder({"resource_key": resource_key}).build() -# We create a FlowData object from the pipeline -# this is used to add evidence to and then process +# The pipeline is closed at the end of the with block, which releases the +# native data it holds. +with pipeline: + # We create a FlowData object from the pipeline + # this is used to add evidence to and then process. It is closed at the + # end of its with block, which releases the results of the detection. + with pipeline.create_flowdata() as flowdata1: -flowdata1 = pipeline.create_flowdata() + # Here we add a User-Agent of a desktop as evidence -# Here we add a User-Agent of a desktop as evidence + desktop_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" -desktop_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" + flowdata1.evidence.add("header.user-agent", desktop_ua) -flowdata1.evidence.add("header.user-agent", desktop_ua) + # Now we process the FlowData -# Now we process the FlowData + flowdata1.process() -flowdata1.process() + # To check whether the User-Agent is a mobile device we look at the ismobile property + # inside the Device Detection Engine -# To check whether the User-Agent is a mobile device we look at the ismobile property -# inside the Device Detection Engine + # first we check if this has a meaningful result -# first we check if this has a meaningful result + print("Is User-Agent " + desktop_ua + " a mobile? ") + if flowdata1.device.ismobile.has_value(): + print(flowdata1.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata1.device.ismobile.no_value_message()) -print("Is User-Agent " + desktop_ua + " a mobile? ") -if flowdata1.device.ismobile.has_value(): - print(flowdata1.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata1.device.ismobile.no_value_message()) + # Now we do the same with a new User-Agent, this time from an iphone -# Now we do the same with a new User-Agent, this time from an iphone + with pipeline.create_flowdata() as flowdata2: -flowdata2 = pipeline.create_flowdata() + iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" -iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" + flowdata2.evidence.add("header.user-agent", iphone_ua) -flowdata2.evidence.add("header.user-agent", iphone_ua) + flowdata2.process() -flowdata2.process() - -print("Is User-Agent " + iphone_ua + " a mobile device?: ") -if flowdata2.device.ismobile.has_value(): - print(flowdata2.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata2.device.ismobile.no_value_message()) + print("Is User-Agent " + iphone_ua + " a mobile device?: ") + if flowdata2.device.ismobile.has_value(): + print(flowdata2.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata2.device.ismobile.no_value_message()) diff --git a/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py b/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py index b7064b14..a8de1b36 100644 --- a/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py +++ b/device-detection/fiftyone_devicedetection_examples/performance-tests/test_performance.py @@ -48,16 +48,18 @@ def detect(app, pipeline, user_agents): # in flowdata.process(). This mock-request doesn't set it, so we set it manually request.remote_addr = "127.0.0.1" - flowdata = pipeline.create_flowdata() + # The FlowData is closed when the function returns, which releases + # the results of the detection straight away. + with pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information) - flowdata.evidence.add_from_dict(webevidence(request)) + # Add any information from the request (headers, cookies and additional + # client side provided information) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata - flowdata.process() + # Process the flowdata + flowdata.process() - return getValueHelper(flowdata, "device", "ismobile") + return getValueHelper(flowdata, "device", "ismobile") def benchmark(data_file, user_agents_file, output=False): app = Flask(__name__) @@ -73,13 +75,16 @@ def benchmark(data_file, user_agents_file, output=False): settings = { "add_javascript_builder": False }, ).build() - with open(user_agents_file) as f: - user_agents_list = [ua.rstrip() for ua in f] - user_agents = iter(user_agents_list) + # The pipeline is closed once the detections have been timed, which + # releases the native data it holds. + with pipeline: + with open(user_agents_file) as f: + user_agents_list = [ua.rstrip() for ua in f] + user_agents = iter(user_agents_list) - print(f"Benchmarking with {len(user_agents_list)} user agents", file=sys.stderr) - environment = {"detect": detect, "app": app, "pipeline": pipeline, "user_agents": user_agents} - time = timeit("detect(app, pipeline, user_agents)", globals=environment, number=len(user_agents_list)) + print(f"Benchmarking with {len(user_agents_list)} user agents", file=sys.stderr) + environment = {"detect": detect, "app": app, "pipeline": pipeline, "user_agents": user_agents} + time = timeit("detect(app, pipeline, user_agents)", globals=environment, number=len(user_agents_list)) if output: json.dump({ diff --git a/device-detection/fiftyone_devicedetection_examples/setup.py b/device-detection/fiftyone_devicedetection_examples/setup.py index 15fa344c..e5ab4dbf 100644 --- a/device-detection/fiftyone_devicedetection_examples/setup.py +++ b/device-detection/fiftyone_devicedetection_examples/setup.py @@ -65,7 +65,9 @@ def read(file_name): # named here so that is visible to anyone reading them. # fiftyone_devicedetection is still required because the on-premise # examples in this same package use the on-premise engine. - install_requires=["fiftyone_devicedetection_cloud", "fiftyone_devicedetection", "flask", "flask-unittest", "json5", "ruamel.yaml"], + # fiftyone_pipeline_core 4.5.42 is the first version with close(), which + # the examples use. + install_requires=["fiftyone_devicedetection_cloud", "fiftyone_devicedetection", "fiftyone_pipeline_core>=4.5.42", "flask", "flask-unittest", "json5", "ruamel.yaml"], license="EUPL-1.2", classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index d5fd2228..4fd126fc 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -234,15 +234,18 @@ def run(self, data_file, license_key, interactive, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - # and output the results - ExampleUtils.check_data_file(pipeline, logger) - if (ExampleUtils.get_data_file_tier(pipeline.get_element("device")) == "Lite"): - logger.log("error", - "Will not download an 'Enterprise' data file over the top of " - "a 'Lite' data file, please supply another location.") - raise Exception("File supplied has wrong data tier") - logger.log("info", "Existing data file will be replaced with downloaded data file") - logger.log("info", f"Existing data file will be copied to {copy_data_file_name}") + # The pipeline is closed at the end of the with block, which + # releases the native data it holds before the file is replaced. + with pipeline: + # and output the results + ExampleUtils.check_data_file(pipeline, logger) + if (ExampleUtils.get_data_file_tier(pipeline.get_element("device")) == "Lite"): + logger.log("error", + "Will not download an 'Enterprise' data file over the top of " + "a 'Lite' data file, please supply another location.") + raise Exception("File supplied has wrong data tier") + logger.log("info", "Existing data file will be replaced with downloaded data file") + logger.log("info", f"Existing data file will be copied to {copy_data_file_name}") # do we really want to do this if (interactive): @@ -295,41 +298,44 @@ def run(self, data_file, license_key, interactive, logger, output): data_update_product_type="V4Enterprise", ).add_logger(logger).build() - - # thread blocks till update checking is complete - or if there is an - # exception we don't get this far - update_event.wait() - output(f"Update on start-up complete - status - {update_event.status}") - - if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: - - output("Modifying downloaded file to trigger reload - please wait for that" - " to complete") - - # wait for the dataUpdateService to notify us that it has updated - update_event.clear() - - # it's the same file but changing the file metadata will trigger reload, - # demonstrating that if you download a new file and replace the - # existing one, then it will be loaded - now = datetime.now().timestamp() - try: - os.utime(data_file, (now, now)) - except: - raise Exception("Could not modify file time, abandoning " - "example") - - if update_event.wait(120): - output(f"Update on file modification complete, status: {update_event.status}") - else: - output("Update on file modification timed out") - else: - logger.log("error", "Auto update was not successful, abandoning example") - error_message = f"Auto update failed: {update_event.status}" - if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: - output(error_message) + # The pipeline is closed at the end of the with block, which stops + # the automatic updates and the file system watcher, and releases + # the native data it holds. + with pipeline: + # thread blocks till update checking is complete - or if there is an + # exception we don't get this far + update_event.wait() + output(f"Update on start-up complete - status - {update_event.status}") + + if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: + + output("Modifying downloaded file to trigger reload - please wait for that" + " to complete") + + # wait for the dataUpdateService to notify us that it has updated + update_event.clear() + + # it's the same file but changing the file metadata will trigger reload, + # demonstrating that if you download a new file and replace the + # existing one, then it will be loaded + now = datetime.now().timestamp() + try: + os.utime(data_file, (now, now)) + except: + raise Exception("Could not modify file time, abandoning " + "example") + + if update_event.wait(120): + output(f"Update on file modification complete, status: {update_event.status}") + else: + output("Update on file modification timed out") else: - raise Exception(error_message) + logger.log("error", "Auto update was not successful, abandoning example") + error_message = f"Auto update failed: {update_event.status}" + if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: + output(error_message) + else: + raise Exception(error_message) output("Finished Example") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py index f640b1d8..9db313bd 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/failuretomatch.py @@ -33,46 +33,49 @@ performance_profile = 'MaxPerformance', auto_update=False).build() -# We create a FlowData object from the pipeline -# this is used to add evidence to and then process +# The pipeline is closed at the end of the with block, which releases the +# native data it holds. +with pipeline: + # We create a FlowData object from the pipeline + # this is used to add evidence to and then process. It is closed at the + # end of its with block, which releases the results of the detection. + with pipeline.create_flowdata() as flowdata1: -flowdata1 = pipeline.create_flowdata() + # Here we add a User-Agent of an iphone as evidence -# Here we add a User-Agent of an iphone as evidence + iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" -iphone_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" + flowdata1.evidence.add("header.user-agent", iphone_ua) -flowdata1.evidence.add("header.user-agent", iphone_ua) + # Now we process the FlowData -# Now we process the FlowData + flowdata1.process() -flowdata1.process() + # To check whether the User-Agent is a mobile device we look at the ismobile property + # inside the Device Detection Engine -# To check whether the User-Agent is a mobile device we look at the ismobile property -# inside the Device Detection Engine + # first we check if this has a meaningful result -# first we check if this has a meaningful result + print("Is User-Agent " + iphone_ua + " a mobile device?: ") + if flowdata1.device.ismobile.has_value(): + print(flowdata1.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata1.device.ismobile.no_value_message()) -print("Is User-Agent " + iphone_ua + " a mobile device?: ") -if flowdata1.device.ismobile.has_value(): - print(flowdata1.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata1.device.ismobile.no_value_message()) + # Now we do the same with a new User-Agent, this time a corrupted one -# Now we do the same with a new User-Agent, this time a corrupted one + badUA = "--" -badUA = "--" + with pipeline.create_flowdata() as flowdata2: -flowdata2 = pipeline.create_flowdata() + flowdata2.evidence.add("header.user-agent", badUA) -flowdata2.evidence.add("header.user-agent", badUA) + flowdata2.process() -flowdata2.process() - -print("Is User-Agent " + badUA + " a mobile device?: ") -if flowdata2.device.ismobile.has_value(): - print(flowdata2.device.ismobile.value()) -else: - # Output why the value isn't meaningful - print(flowdata2.device.ismobile.no_value_message()) + print("Is User-Agent " + badUA + " a mobile device?: ") + if flowdata2.device.ismobile.has_value(): + print(flowdata2.device.ismobile.value()) + else: + # Output why the value isn't meaningful + print(flowdata2.device.ismobile.no_value_message()) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 946f06bd..3675fee2 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -68,11 +68,16 @@ def run(self, data_file, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. An application normally builds + # one pipeline when it starts, uses it for every detection, and + # closes it when it stops. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) - # carry out some sample detections - for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + # carry out some sample detections + for values in EVIDENCE_VALUES: + self.analyseEvidence(values, pipeline, output) def analyseEvidence(self, evidence, pipeline, output): @@ -83,42 +88,45 @@ def analyseEvidence(self, evidence, pipeline, output): # and usually consists of a number of HTTP Header field # values, in this case represented by a dictionary of header # name/value entries. - data = pipeline.create_flowdata() + # Closing the FlowData, which the with statement does at the end of + # the block, releases the results of the detection straight away, + # rather than when the garbage collector next runs. + with pipeline.create_flowdata() as data: - message = [] + message = [] - # List the evidence - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") + # List the evidence + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") - output("".join(message)) - - # Add the evidence values to the flow data - data.evidence.add_from_dict(evidence) - - # Process the flow data. - data.process() - - message = [] - message.append("Results:\n") - - # Now that it's been processed, the flow data will have - # been populated with the result. In this case, we want - # information about the device, which we can get by - # asking for a result matching named "device" - device = data.device - - # Display the results of the detection, which are called - # device properties. See the property dictionary at - # https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence - # for details of all available properties. - self.outputValue("Mobile Device", device.ismobile, message) - self.outputValue("Platform Name", device.platformname, message) - self.outputValue("Platform Version", device.platformversion, message) - self.outputValue("Browser Name", device.browsername, message) - self.outputValue("Browser Version", device.browserversion, message) - output("".join(message)) + output("".join(message)) + + # Add the evidence values to the flow data + data.evidence.add_from_dict(evidence) + + # Process the flow data. + data.process() + + message = [] + message.append("Results:\n") + + # Now that it's been processed, the flow data will have + # been populated with the result. In this case, we want + # information about the device, which we can get by + # asking for a result matching named "device" + device = data.device + + # Display the results of the detection, which are called + # device properties. See the property dictionary at + # https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence + # for details of all available properties. + self.outputValue("Mobile Device", device.ismobile, message) + self.outputValue("Platform Name", device.platformname, message) + self.outputValue("Platform Version", device.platformversion, message) + self.outputValue("Browser Name", device.browsername, message) + self.outputValue("Browser Version", device.browserversion, message) + output("".join(message)) def outputValue(self, name, value, message): # Individual result values have a wrapper called diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py index d382e6f2..a8e1e3ce 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/app.py @@ -102,7 +102,10 @@ def run(self): # The port can be set with the PORT environment variable, as it can # for the cloud example, so the example can run next to other services. - GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) + # The pipeline lives as long as the app, and is closed when the server + # stops, which releases the native data it holds. + with GettingStartedWeb.pipeline: + GettingStartedWeb.app.run(port=int(os.environ.get("PORT", 5001))) # First we make a JSON route that will be called from the client side and will return # a JSON encoded property database using any additional evidence provided by the client @@ -111,21 +114,23 @@ def run(self): @app.route('/json', methods=['POST']) def jsonroute(): - # Create the flowdata object for the JSON route - flowdata = GettingStartedWeb.pipeline.create_flowdata() + # Create the flowdata object for the JSON route. It is closed when the + # route returns, which releases the results of the detection straight + # away rather than when the garbage collector next runs. + with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information) + # Add any information from the request (headers, cookies and additional + # client side provided information) - flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata + # Process the flowdata - flowdata.process() + flowdata.process() - # Return the JSON from the JSONBundler engine + # Return the JSON from the JSONBundler engine - return json.dumps(flowdata.jsonbundler.json) + return json.dumps(flowdata.jsonbundler.json) # Next we serve the client-side JavaScript from its own route. The other # Pipeline APIs ship a web integration that intercepts '/51Degrees.core.js' @@ -136,26 +141,27 @@ def jsonroute(): @app.route('/51Degrees.core.js') def core_js(): - # Create the flowdata object for the JavaScript route - flowdata = GettingStartedWeb.pipeline.create_flowdata() + # Create the flowdata object for the JavaScript route. It is closed + # when the route returns. + with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information). Query parameters are included, so - # the per-request 'fod-js-enable-cookies' override is picked up here and - # applied by the JavaScriptBuilder engine. + # Add any information from the request (headers, cookies and additional + # client side provided information). Query parameters are included, so + # the per-request 'fod-js-enable-cookies' override is picked up here and + # applied by the JavaScriptBuilder engine. - flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata + # Process the flowdata - flowdata.process() + flowdata.process() - # Return the JavaScript from the JavaScriptBuilder engine + # Return the JavaScript from the JavaScriptBuilder engine - response = make_response(flowdata.javascriptbuilder.javascript) - response.headers["Content-Type"] = "application/x-javascript" + response = make_response(flowdata.javascriptbuilder.javascript) + response.headers["Content-Type"] = "application/x-javascript" - return response + return response # In the main route we dynamically update the screen's device property display # using the above JSON route @@ -164,36 +170,37 @@ def core_js(): @app.route('/') def server(): - # Create the flowdata object for the JSON route - flowdata = GettingStartedWeb.pipeline.create_flowdata() + # Create the flowdata object for the page. It is closed when the route + # returns, once the page has been rendered from it. + with GettingStartedWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and additional - # client side provided information) + # Add any information from the request (headers, cookies and additional + # client side provided information) - flowdata.evidence.add_from_dict(webevidence(request)) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata + # Process the flowdata - flowdata.process() + flowdata.process() - response = make_response() + response = make_response() - # Some browsers require that extra HTTP headers are explicitly - # requested. So set whatever headers are required by the browser in - # order to return the evidence needed by the pipeline. - # More info on this can be found at - # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=server + # Some browsers require that extra HTTP headers are explicitly + # requested. So set whatever headers are required by the browser in + # order to return the evidence needed by the pipeline. + # More info on this can be found at + # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_web-app.py&utm_term=server - set_response_header(flowdata, response) + set_response_header(flowdata, response) - # Generate the HTML - response.set_data(render_template( - 'index.html', - data=flowdata, - utils=ExampleUtils, - response=response)) + # Generate the HTML + response.set_data(render_template( + 'index.html', + data=flowdata, + utils=ExampleUtils, + response=response)) - return response + return response # Typically, something like this will not be necessary. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 5daeb284..6a223d00 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -88,83 +88,89 @@ def run(self, data_file, show_descs, logger, output): # We want to show the matching evidence characters as part of this example, so we have to set # this flag to true. update_matched_useragent = True).add_logger(logger).build() + + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) + + # Closing the FlowData, which the with statement does at the end + # of the block, releases the results of the detection straight + # away, rather than when the garbage collector next runs. + with pipeline.create_flowdata() as data: + + # Process a single evidence to retrieve the values + # associated with the user-agent and other evidence such as sec-ch-* for the + # selected properties. + data.evidence.add_from_dict(self.Evidence) + data.process() + + device = data.device + + output("--- Compare evidence with what was matched ---\n") + output("Evidence") + # output the evidence in reverse value length order + for entry in sorted(self.Evidence.items(), key=lambda item: len(item[1]), reverse=True): + output(f" {entry[0]}: {entry[1]}") + # Obtain the matched User-Agents: the matched substrings in the + # User-Agents are separated with underscores - output in forward length order. + output("Matches") + for entry in sorted(device.useragents.value(), key=lambda item: len(item)): + output(f" Matched User-Agent: {entry}") + + output("") + + + output("--- Listing all available properties, by component, by property name ---") + output("For a discussion of what the match properties mean, see: https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=match-properties-discussion#DeviceDetection_Hash_DataSetProduction_Performance\n") + + # get the properties available from the DeviceDetection engine + # which has the key "device". For the sake of illustration we will + # retrieve it indirectly. + hashEngineElementKey = pipeline.get_element("device").datakey + + # retrieve the available properties from the hash engine. The properties + # available depends on + # a) the use of restricted_properties in the builder (see above) + # which controls which properties will be extracted, and also affects + # the performance of extraction + # b) the tier of data file being used. The Lite data file contains fewer + # than 20 of the >200 available properties + availableProperties = dict(pipeline.get_properties()[hashEngineElementKey]) - ExampleUtils.check_data_file(pipeline, logger) - - data = pipeline.create_flowdata() - - # Process a single evidence to retrieve the values - # associated with the user-agent and other evidence such as sec-ch-* for the - # selected properties. - data.evidence.add_from_dict(self.Evidence) - data.process() - - device = data.device - - output("--- Compare evidence with what was matched ---\n") - output("Evidence") - # output the evidence in reverse value length order - for entry in sorted(self.Evidence.items(), key=lambda item: len(item[1]), reverse=True): - output(f" {entry[0]}: {entry[1]}") - # Obtain the matched User-Agents: the matched substrings in the - # User-Agents are separated with underscores - output in forward length order. - output("Matches") - for entry in sorted(device.useragents.value(), key=lambda item: len(item)): - output(f" Matched User-Agent: {entry}") - - output("") - - - output("--- Listing all available properties, by component, by property name ---") - output("For a discussion of what the match properties mean, see: https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=match-properties-discussion#DeviceDetection_Hash_DataSetProduction_Performance\n") - - # get the properties available from the DeviceDetection engine - # which has the key "device". For the sake of illustration we will - # retrieve it indirectly. - hashEngineElementKey = pipeline.get_element("device").datakey - - # retrieve the available properties from the hash engine. The properties - # available depends on - # a) the use of restricted_properties in the builder (see above) - # which controls which properties will be extracted, and also affects - # the performance of extraction - # b) the tier of data file being used. The Lite data file contains fewer - # than 20 of the >200 available properties - availableProperties = dict(pipeline.get_properties()[hashEngineElementKey]) - - # create a Map keyed on the component name of the properties available - # components being hardware, browser, OS and Crawler. - def get_component(property): - return property[1]["component"] - - availableProperties = dict(sorted(availableProperties.items(),key=lambda item: get_component(item))) - categoryMap = groupby(availableProperties.items(), get_component) - # iterate the map created above - for component, properties in categoryMap: - output(component) - for propertyKey, property in properties: - propertyName = property["name"] - propertyDescription = property["description"] + # create a Map keyed on the component name of the properties available + # components being hardware, browser, OS and Crawler. + def get_component(property): + return property[1]["component"] + + availableProperties = dict(sorted(availableProperties.items(),key=lambda item: get_component(item))) + categoryMap = groupby(availableProperties.items(), get_component) + # iterate the map created above + for component, properties in categoryMap: + output(component) + for propertyKey, property in properties: + propertyName = property["name"] + propertyDescription = property["description"] - # while we get the available properties and their metadata from the - # pipeline we get the values for the last detection from flowData - value = device[propertyKey] + # while we get the available properties and their metadata from the + # pipeline we get the values for the last detection from flowData + value = device[propertyKey] - # output property names, values and descriptions - # some property values are lists. - if value.has_value() and isinstance(value.value(), list): - output(f" {propertyName}: {len(value.value())} Values") - for item in value.value(): - output(f" : {item}") + # output property names, values and descriptions + # some property values are lists. + if value.has_value() and isinstance(value.value(), list): + output(f" {propertyName}: {len(value.value())} Values") + for item in value.value(): + output(f" : {item}") - else: - output(f" {propertyName}: {value.value()}") + else: + output(f" {propertyName}: {value.value()}") - if (show_descs == True): - output(f" {propertyDescription}") + if (show_descs == True): + output(f" {propertyDescription}") - output() - logger.log("info", "Finished Match Metrics Example") + output() + logger.log("info", "Finished Match Metrics Example") # Evidence values from a windows 11 device using a browser # that supports User-Agent Client Hints. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 2df9d104..5e605158 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -76,10 +76,14 @@ def run(self, data_file, logger, output): # Inhibit auto-update of the data file for this example auto_update = False, licence_keys = "").add_logger(logger).build() - self.outputProperties(pipeline.get_element("device"), output) - self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + self.outputProperties(pipeline.get_element("device"), output) + self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) + + ExampleUtils.check_data_file(pipeline, logger) def outputEvidenceKeyDetails(self, engine, output): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 7a90c401..500fdba7 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -99,40 +99,43 @@ def run(self, data_file, evidence_yaml, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - records = 0 - yaml = YAML() - # Replace tab characters as they are not valid in unquoted YAML values - # and some evidence files contain them in user-agent strings. - content = evidence_yaml.read().replace('\t', ' ') - yaml_data = yaml.load_all(content) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + records = 0 + yaml = YAML() + # Replace tab characters as they are not valid in unquoted YAML values + # and some evidence files contain them in user-agent strings. + content = evidence_yaml.read().replace('\t', ' ') + yaml_data = yaml.load_all(content) - try: - # Keep going as long as we have more document records. - for evidence in yaml_data: - # Output progress. - records = records + 1 - if (records % 100 == 0): - logger.log("info", f"Processed {records} records") + try: + # Keep going as long as we have more document records. + for evidence in yaml_data: + # Output progress. + records = records + 1 + if (records % 100 == 0): + logger.log("info", f"Processed {records} records") - # write the yaml document separator - print("---", file = output) - # Pass the record to the pipeline as evidence so that it can be analyzed - headers = {} - for key in evidence: - # Convert to string - YAML may parse values like ?1 as booleans - # Keys already have header. prefix in the YAML file - headers[key] = str(evidence[key]) + # write the yaml document separator + print("---", file = output) + # Pass the record to the pipeline as evidence so that it can be analyzed + headers = {} + for key in evidence: + # Convert to string - YAML may parse values like ?1 as booleans + # Keys already have header. prefix in the YAML file + headers[key] = str(evidence[key]) - self.analyseEvidence(headers, pipeline, output, yaml) - except BaseException as err: - # We can't read the evidence values, so cant write them to the output. Will just - # have to skip this entry. - logger.log("error", err) + self.analyseEvidence(headers, pipeline, output, yaml) + except BaseException as err: + # We can't read the evidence values, so cant write them to the output. Will just + # have to skip this entry. + logger.log("error", err) - # write the yaml document end marker - print("...", file = output) + # write the yaml document end marker + print("...", file = output) - ExampleUtils.check_data_file(pipeline, logger) + ExampleUtils.check_data_file(pipeline, logger) def analyseEvidence(self, evidence, pipeline, output, yaml): # FlowData is a data structure that is used to convey information required for @@ -140,46 +143,50 @@ def analyseEvidence(self, evidence, pipeline, output, yaml): # Information required for detection is called "evidence" and usually consists # of a number of HTTP Header field values, in this case represented by a # dictionary of header name/value entries. - data = pipeline.create_flowdata() - # Add the evidence values to the flow data - data.evidence.add_from_dict(evidence) - # Process the flow data. - data.process() + # Closing the FlowData, which the with statement does at the end of + # the block, releases the results of the detection straight away. + # Without it each record's results would stay in memory until the + # garbage collector next runs. + with pipeline.create_flowdata() as data: + # Add the evidence values to the flow data + data.evidence.add_from_dict(evidence) + # Process the flow data. + data.process() - device = data.device + device = data.device - values = {} - # Add the evidence values to the output - for key in evidence: - values[key] = evidence[key] - # Now add the values that we want to store against the record. - values["device.ismobile"] = device.ismobile.value() if device.ismobile.has_value() else "Unknown" - values["device.platformname"] = ExampleUtils.get_human_readable(device, "platformname") - values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") - values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") - values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") - values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") - values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") - values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") - values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") - values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") - values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") - values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") - values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") - # DeviceId is a unique identifier for the combination of hardware, operating - # system, browser and crawler that has been detected. - # Our device detection solution uses machine learning to find the optimal - # way to identify devices based on the real-world evidence values that we - # observe each day. - # As this changes over time, the result of detection can potentially change - # as well. By storing the device id, we can use this as a lookup in future - # rather than performing detection with the original evidence again. - # Do this by passing an evidence entry with: - # key = query.51D_ProfileIds - # value = [the device id] - # This is much faster and avoids the potential for getting a different - # result. - values["device.deviceid"] = ExampleUtils.get_human_readable(device, "deviceid") + values = {} + # Add the evidence values to the output + for key in evidence: + values[key] = evidence[key] + # Now add the values that we want to store against the record. + values["device.ismobile"] = device.ismobile.value() if device.ismobile.has_value() else "Unknown" + values["device.platformname"] = ExampleUtils.get_human_readable(device, "platformname") + values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") + values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") + values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") + values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") + values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") + values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") + values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") + values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") + values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") + values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") + values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") + # DeviceId is a unique identifier for the combination of hardware, operating + # system, browser and crawler that has been detected. + # Our device detection solution uses machine learning to find the optimal + # way to identify devices based on the real-world evidence values that we + # observe each day. + # As this changes over time, the result of detection can potentially change + # as well. By storing the device id, we can use this as a lookup in future + # rather than performing detection with the original evidence again. + # Do this by passing an evidence entry with: + # key = query.51D_ProfileIds + # value = [the device id] + # This is much faster and avoids the potential for getting a different + # result. + values["device.deviceid"] = ExampleUtils.get_human_readable(device, "deviceid") yaml.dump(values, output) def main(argv): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index 271ce134..d795b548 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -62,24 +62,26 @@ def process_user_agent(user_agent): - # First we create the flowdata using the global pipeline - flowdata = pipeline.create_flowdata() # pylint: disable=used-before-assignment + # First we create the flowdata using the global pipeline. Closing it, + # which the with statement does when the function returns, releases the + # results of the detection straight away. + with pipeline.create_flowdata() as flowdata: # pylint: disable=used-before-assignment - # Here we add the user agent as evidence - flowdata.evidence.add("header.user-agent", user_agent) + # Here we add the user agent as evidence + flowdata.evidence.add("header.user-agent", user_agent) - # We process the flowdata to get the results - flowdata.process() + # We process the flowdata to get the results + flowdata.process() - # To check whether the User-Agent is a mobile device we look at the ismobile - # property inside the Device Detection Engine + # To check whether the User-Agent is a mobile device we look at the ismobile + # property inside the Device Detection Engine - # first we check if this has a meaningful result + # first we check if this has a meaningful result - if flowdata.device.ismobile.has_value(): - return flowdata.device.ismobile.value() - else: - return None + if flowdata.device.ismobile.has_value(): + return flowdata.device.ismobile.value() + else: + return None def process_user_agent_list(data_file, user_agent_list, list_number, output, skip=False): global pipeline @@ -92,23 +94,26 @@ def process_user_agent_list(data_file, user_agent_list, list_number, output, ski usage_sharing=False, auto_update=False).build() - results = { - "mobile": 0, - "notmobile": 0, - "unknown": 0 - } - for user_agent in user_agent_list: - if skip: - break - result = process_user_agent(user_agent[0]) - if(result == None): - results["unknown"] += 1 - if(result == True): - results["mobile"] += 1 - if(result == False): - results["notmobile"] += 1 - - output.put(results, list_number) + # The pipeline is closed at the end of the with block, which releases + # the native data it holds. + with pipeline: + results = { + "mobile": 0, + "notmobile": 0, + "unknown": 0 + } + for user_agent in user_agent_list: + if skip: + break + result = process_user_agent(user_agent[0]) + if(result == None): + results["unknown"] += 1 + if(result == True): + results["mobile"] += 1 + if(result == False): + results["notmobile"] += 1 + + output.put(results, list_number) # Run the process def run(data_file, skip = False): diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py index 6cbbbfd7..51bcff0c 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/shareusage.py @@ -130,46 +130,51 @@ def run(self, evidence_yaml): # Build a pipeline with only the ShareUsage element pipeline = PipelineBuilder().add(share_usage_element).build() - print("Pipeline built successfully with ShareUsage element") - - # Load YAML evidence - yaml = YAML() - yaml_data = yaml.load_all(evidence_yaml) - - # Process evidence records - count = 0 - for evidence in yaml_data: - if count >= RECORDS_TO_PROCESS: - break - - # Filter to only header.* entries - data = filter_evidence( - {k: str(v) for k, v in evidence.items()}, - "header.") - - # Add a client IP address to the evidence for deduplication purposes - # (see class-level note). In a real web application, this would come - # from the request (e.g., REMOTE_ADDR or X-Forwarded-For header) - data[EVIDENCE_CLIENTIP_KEY] = generate_random_ip() - - # Add the usage-from identifier so 51Degrees knows the source of this data. - # This appears as
    YourCompanyName
    in the XML. - data[EVIDENCE_USAGE_FROM_KEY] = USAGE_FROM_VALUE - - # Create flow data and process - flow_data = pipeline.create_flowdata() - flow_data.evidence.add_from_dict(data) - flow_data.process() - - count += 1 - if count % 10 == 0: - print(f"Processed {count} records") - - print(f"Finished processing {count} records") - print("Waiting for share usage to complete sending...") - - # Give the background thread time to send any remaining data - time.sleep(5) + # The pipeline is closed at the end of the with block, which waits + # for any usage data being sent and then stops the threads that send + # it. + with pipeline: + print("Pipeline built successfully with ShareUsage element") + + # Load YAML evidence + yaml = YAML() + yaml_data = yaml.load_all(evidence_yaml) + + # Process evidence records + count = 0 + for evidence in yaml_data: + if count >= RECORDS_TO_PROCESS: + break + + # Filter to only header.* entries + data = filter_evidence( + {k: str(v) for k, v in evidence.items()}, + "header.") + + # Add a client IP address to the evidence for deduplication purposes + # (see class-level note). In a real web application, this would come + # from the request (e.g., REMOTE_ADDR or X-Forwarded-For header) + data[EVIDENCE_CLIENTIP_KEY] = generate_random_ip() + + # Add the usage-from identifier so 51Degrees knows the source of this data. + # This appears as
    YourCompanyName
    in the XML. + data[EVIDENCE_USAGE_FROM_KEY] = USAGE_FROM_VALUE + + # Create flow data and process. The with statement closes it + # once it has been processed. + with pipeline.create_flowdata() as flow_data: + flow_data.evidence.add_from_dict(data) + flow_data.process() + + count += 1 + if count % 10 == 0: + print(f"Processed {count} records") + + print(f"Finished processing {count} records") + print("Waiting for share usage to complete sending...") + + # Give the background thread time to send any remaining data + time.sleep(5) print("Done!") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py index 570ab57e..789c2d59 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/app.py @@ -73,7 +73,10 @@ def build(self, config, logger): return self def run(self): - UserAgentClientHintsWeb.app.run(port=5001) + # The pipeline lives as long as the app, and is closed when the server + # stops, which releases the native data it holds. + with UserAgentClientHintsWeb.pipeline: + UserAgentClientHintsWeb.app.run(port=5001) # The main route performs device detection and renders the results. On the # second request, any additional client hints headers requested via the @@ -83,33 +86,35 @@ def run(self): @app.route('/') def server(): - # Create the flowdata object for the device detection - flowdata = UserAgentClientHintsWeb.pipeline.create_flowdata() + # Create the flowdata object for the device detection. It is closed + # when the route returns, once the page has been rendered from it, + # which releases the results of the detection straight away. + with UserAgentClientHintsWeb.pipeline.create_flowdata() as flowdata: - # Add any information from the request (headers, cookies and any other - # additional information) - flowdata.evidence.add_from_dict(webevidence(request)) + # Add any information from the request (headers, cookies and any other + # additional information) + flowdata.evidence.add_from_dict(webevidence(request)) - # Process the flowdata - flowdata.process() + # Process the flowdata + flowdata.process() - response = make_response() + response = make_response() - # Some browsers require that extra HTTP headers are explicitly - # requested. So set whatever headers are required by the browser in - # order to return the evidence needed by the pipeline. - # More info on this can be found at - # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-useragentclienthints_web-app.py&utm_term=server - set_response_header(flowdata, response) + # Some browsers require that extra HTTP headers are explicitly + # requested. So set whatever headers are required by the browser in + # order to return the evidence needed by the pipeline. + # More info on this can be found at + # https://51degrees.com/blog/user-agent-client-hints?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-useragentclienthints_web-app.py&utm_term=server + set_response_header(flowdata, response) - # Generate the HTML - response.set_data(render_template( - 'index.html', - data=flowdata, - utils=ExampleUtils, - response=response)) + # Generate the HTML + response.set_data(render_template( + 'index.html', + data=flowdata, + utils=ExampleUtils, + response=response)) - return response + return response # Typically, something like this will not be necessary. # The device detection API will accept an absolute or relative path for the data file. diff --git a/device-detection/fiftyone_devicedetection_examples/tests/conftest.py b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py index 70374c6a..a997f5c7 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_onpremisegettingstartedweb.py @@ -39,6 +39,13 @@ class OnPremiseGettingStartedWebTests(flask_unittest.ClientTestCase): config = GettingStartedWeb.build_config() app = GettingStartedWeb().build(config, logger).app + # The example's pipeline lives as long as its app, so it is closed once + # the tests are finished with the app. + @classmethod + def tearDownClass(cls): + GettingStartedWeb.pipeline.close() + super().tearDownClass() + def test_onpremise_getting_started_web(self, client): response = client.get('/') self.assertEqual(200, response.status_code) diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 6b005879..0b6222c0 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -22,6 +22,12 @@ deps = -e../fiftyone_devicedetection_cloud/ -e../fiftyone_devicedetection_onpremise/ -e../fiftyone_devicedetection_shared/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pwsh -ExecutionPolicy Bypass -File ../ci/build-onpremise-c-module.ps1 {[common]commands} diff --git a/device-detection/fiftyone_devicedetection_onpremise/setup.py b/device-detection/fiftyone_devicedetection_onpremise/setup.py index 0af5631d..83b58226 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/setup.py +++ b/device-detection/fiftyone_devicedetection_onpremise/setup.py @@ -261,5 +261,12 @@ def stage_native_source(): py_modules=["fiftyone_devicedetection_onpremise"], packages=["fiftyone_devicedetection_onpremise"], package_dir={"": "src"}, - install_requires=["fiftyone_devicedetection_shared", "fiftyone_pipeline_engines_fiftyone"], + install_requires=[ + "fiftyone_devicedetection_shared", + "fiftyone_pipeline_engines_fiftyone", + # The first versions with close(), which releases the native memory + # this package holds. + "fiftyone_pipeline_core>=4.5.42", + "fiftyone_pipeline_engines>=4.5.42", + ], ) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py index bb8b3b64..db622b40 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py @@ -41,3 +41,10 @@ 'on-premise device detection engine. Its results ' 'read from native memory, which is not safe to ' 'share between requests.') + +ENGINE_CLOSED = ('The on-premise device detection engine has been closed, ' + 'so it cannot process evidence. Build a new pipeline.') + +RESULTS_CLOSED = ('These device detection results have been closed, so ' + 'their properties cannot be read. Read the values needed ' + 'before closing the FlowData.') diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index d8de7ec7..632751ae 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -403,6 +403,20 @@ def on_registration(self, pipeline): super(DeviceDetectionOnPremise, self).on_registration(pipeline) self.init_engine() + def close(self): + + """! + + Stop the data file updates and release the native engine. Its data + is freed once no results read from it are still held, so close + every FlowData the engine processed as well. Closing the pipeline + closes the engine. + + """ + + super(DeviceDetectionOnPremise, self).close() + self.engine = None + def process_internal(self, flow_data): """! @@ -414,6 +428,9 @@ def process_internal(self, flow_data): """ + if getattr(self, "engine", None) is None: + raise Exception(ENGINE_CLOSED) + evidence = EvidenceDeviceDetectionSwig() for key, value in flow_data.evidence.get_all().items(): diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index b1b4e8dd..17987a59 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -25,6 +25,8 @@ from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService +from .constants import RESULTS_CLOSED + class OnPremiseMissingPropertyService(MissingPropertyService): def check(self, key, flow_element): @@ -59,6 +61,33 @@ def __init__(self, flow_element, swig_results): self.missing_property_service = OnPremiseMissingPropertyService() + def close(self): + + """! + Release the native results, which keep the engine's data in memory + while they are held, even after the engine is closed. Values already + read are unaffected, and reading a property afterwards raises an + exception. Closing the FlowData closes its results. + """ + + self.swig_results = None + + def get(self, key): + + """! + Get a property value, once the results are known to be open + @type key: string + @param key: The key of the property to retrieve + @returns mixed + """ + + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) + + return super(SwigData, self).get(key) + def get_internal(self, key): """! diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py index ea5a031f..e1ac9944 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py @@ -68,8 +68,10 @@ def test_on_premise_data_update_url(self): data_update_verify_md5=False, data_update_url=f"http://127.0.0.1:{self.port}" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -85,8 +87,10 @@ def test_on_premise_data_old_property_data_file_update_base_url(self): data_update_verify_md5=False, data_file_update_base_url=f"http://127.0.0.1:{self.port}" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -106,7 +110,9 @@ def test_on_premise_data_update_url_config(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -126,7 +132,9 @@ def test_on_premise_data_config_old_property_data_file_update_base_url(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -142,8 +150,10 @@ def test_on_premise_data_update_verify_md5(self): data_update_verify_md5=True, data_update_url=f"http://127.0.0.1:{self.port}/md5" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -159,8 +169,10 @@ def test_on_premise_data_update_verify_md5_old_property(self): verify_md5=True, data_update_url=f"http://127.0.0.1:{self.port}/md5" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -180,7 +192,9 @@ def test_on_premise_data_update_verify_md5_config(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -200,7 +214,9 @@ def test_on_premise_data_update_verify_md5_config_old_property(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -217,8 +233,10 @@ def test_on_premise_data_update_use_url_formatter(self): data_update_url=f"http://127.0.0.1:{self.port}/", data_update_use_url_formatter=False ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -238,7 +256,9 @@ def test_on_premise_data_update_use_url_formatter_config(self): ) pipeline = PipelineBuilder().build_from_configuration(config) - pipeline.create_flowdata().process() + self.addCleanup(pipeline.close) + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -253,8 +273,10 @@ def test_on_premise_data_update_status_rate_limit(self): verify_md5=True, data_update_url=f"http://127.0.0.1:{self.port}/status/429" ).build() + self.addCleanup(pipeline.close) - pipeline.create_flowdata().process() + with pipeline.create_flowdata() as flowdata: + flowdata.process() flow_element_data_file = pipeline.flow_elements[0].data_file self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile) @@ -325,7 +347,8 @@ def _init_hash_file_archive(): with gzip.open(archive_file_path, "wb", compresslevel=1) as f_out: shutil.copyfileobj(f_in, f_out) - return hashlib.md5(open(archive_file_path, "rb").read()).hexdigest() + with open(archive_file_path, "rb") as archive: + return hashlib.md5(archive.read()).hexdigest() @staticmethod def clear(): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index 135cc69d..11ed21f6 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -61,6 +61,7 @@ def test_pipeline_builder_shareusage_init(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, licence_keys="").build() + self.addCleanup(pipeline.close) self.assertTrue(pipeline.flow_elements[0].datakey == "shareusage") @@ -71,6 +72,7 @@ def test_on_premise_engine_datafile(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="test", auto_update=True).build() + self.addCleanup(pipeline.close) self.assertTrue(isinstance( pipeline.flow_elements[0].data_file, DeviceDetectionDataFile)) @@ -83,6 +85,7 @@ def test_properties_onpremise(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) properties = pipeline.flow_elements[0].get_properties() @@ -95,6 +98,7 @@ def test_evidencekey_filter_onpremise(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -110,8 +114,10 @@ def test_basic_get_onpremise(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -126,8 +132,10 @@ def test_process_with_no_evidence(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.process() @@ -140,8 +148,10 @@ def test_process_with_empty_user_agent(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", "") @@ -156,8 +166,10 @@ def test_process_with_no_headers(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("irrelevant.evidence", "some-evidence") @@ -172,8 +184,10 @@ def test_process_with_no_useful_headers(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.irrelevant", "some-evidence") @@ -188,8 +202,10 @@ def test_case_insensitive_evidence_keys(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -205,8 +221,10 @@ def test_missing_property_service_not_found_anywhere(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -230,8 +248,10 @@ def test_missing_property_service_not_found_in_current(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -262,8 +282,10 @@ def test_excluded_property(self): data_file_path=data_file, licence_keys="", restricted_properties=["ismobile"]).build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -286,6 +308,7 @@ def test_evidencekey_filter_contains_user_agent(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -300,6 +323,7 @@ def test_evidencekey_filter_contains_device_stock_ua(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -314,6 +338,7 @@ def test_evidencekey_filter_contains_query_params(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -330,6 +355,7 @@ def test_evidencekey_filter_case_insensitive_keys(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -350,6 +376,7 @@ def test_evidencekey_filter_overrides(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( ) @@ -366,8 +393,10 @@ def test_profile_overrides(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("query.header.user-agent", "Some user agent") @@ -384,8 +413,10 @@ def test_profile_overrides_no_headers(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("query.51D_ProfileIds", "12280|17779|17470|18092") @@ -400,8 +431,10 @@ def test_profile_overrides_deviceid(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("query.51D_ProfileIds", "12280-17779-17470-18092") @@ -416,8 +449,10 @@ def test_device_id(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -443,8 +478,10 @@ def test_build_from_config(self): } pipeline = PipelineBuilder().build_from_configuration(config) + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.User-Agent", mobile_ua) @@ -459,8 +496,10 @@ def test_no_element_exists(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.process() @@ -481,8 +520,10 @@ def test_has_value_false(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", "?£$%^&*") @@ -497,8 +538,10 @@ def test_matched_user_agents(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -519,8 +562,10 @@ def test_value_types(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -539,8 +584,10 @@ def test_available_properties(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -561,8 +608,10 @@ def test_engine_init_performance(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -586,8 +635,10 @@ def test_validate_data_true(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", mobile_ua) @@ -613,8 +664,10 @@ def test_validate_data_false(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.evidence.add("header.user-agent", "") @@ -638,30 +691,89 @@ def test_engine_reload(self): pipeline = DeviceDetectionOnPremisePipelineBuilder( data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) fd = pipeline.create_flowdata() + self.addCleanup(fd.close) fd.process() fd.pipeline.get_element("device").engine.refreshData() + def test_flowdata_close_releases_results(self): + """! + Closing the FlowData releases its native results, and reading a + property afterwards raises an exception + """ + + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, usage_sharing=False, licence_keys="").build() + self.addCleanup(pipeline.close) + + with pipeline.create_flowdata() as fd: + fd.evidence.add("header.user-agent", mobile_ua) + fd.process() + ismobile = fd.device.ismobile.value() + + self.assertTrue(ismobile) + self.assertIsNone(fd.device.swig_results) + with self.assertRaises(Exception) as context: + fd.device.ismobile + self.assertIn("closed", str(context.exception)) + + def test_pipeline_close_releases_engine(self): + """! + Closing the pipeline releases the native engine, and processing + afterwards raises an exception + """ + + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, usage_sharing=False, licence_keys="").build() + engine = pipeline.get_element("device") + + pipeline.close() + + self.assertIsNone(engine.engine) + fd = pipeline.create_flowdata() + self.addCleanup(fd.close) + fd.evidence.add("header.user-agent", mobile_ua) + with self.assertRaises(Exception) as context: + fd.process() + self.assertIn("closed", str(context.exception)) + + def test_pipeline_close_stops_updates(self): + """! + Closing the pipeline cancels the engine's next automatic update, + whose timer would otherwise keep the engine alive + """ + + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=data_file, usage_sharing=False, licence_keys="test", + auto_update=True).build() + timer = pipeline.get_element("device").data_file.update_timer + + pipeline.close() + + timer.join(10) + self.assertFalse(timer.is_alive()) + def process_user_agent_list(self, pipeline): results = {"mobile": 0, "not_mobile": 0, "unknown": 0} for user_agent in user_agents: user_agent = user_agent[0] - flowdata = pipeline.create_flowdata() - flowdata.evidence.add("header.user-agent", user_agent) - flowdata.process() - - if flowdata.device.ismobile.has_value(): - if(flowdata.device.ismobile.value()): - results["mobile"] += 1 + with pipeline.create_flowdata() as flowdata: + flowdata.evidence.add("header.user-agent", user_agent) + flowdata.process() + + if flowdata.device.ismobile.has_value(): + if(flowdata.device.ismobile.value()): + results["mobile"] += 1 + else: + results["not_mobile"] += 1 else: - results["not_mobile"] += 1 - else: - results["unknown"] += 1 + results["unknown"] += 1 return results diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py index 0d9b6435..df503004 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_properties.py @@ -42,6 +42,10 @@ .add(deviceDetectionOnPremiseEngine) \ .build() +# Releases the native engine once every test in the module has run. +def tearDownModule(): + pipeline.close() + class PropertyTests(unittest.TestCase): def test_available_properties(self): @@ -50,6 +54,7 @@ def test_available_properties(self): """ flowData = pipeline.create_flowdata() + self.addCleanup(flowData.close) flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() device = flowData.device @@ -75,6 +80,7 @@ def test_match_metrics_description(self): """ flowData = pipeline.create_flowdata() + self.addCleanup(flowData.close) flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() @@ -110,6 +116,7 @@ def test_value_types(self): """ flowData = pipeline.create_flowdata() + self.addCleanup(flowData.close) flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() device = flowData.device diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py index 0b4bea5a..67879404 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_swig.py @@ -39,6 +39,10 @@ .add(deviceDetectionOnPremiseEngine) \ .build() +# Releases the native engine once every test in the module has run. +def tearDownModule(): + pipeline.close() + class SwigTests(unittest.TestCase): # Mock the native code parts so we are only testing the wrapper. diff --git a/device-detection/fiftyone_devicedetection_onpremise/tox.ini b/device-detection/fiftyone_devicedetection_onpremise/tox.ini index 4600cd37..5cdeeca3 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tox.ini +++ b/device-detection/fiftyone_devicedetection_onpremise/tox.ini @@ -22,6 +22,12 @@ deps = -e../fiftyone_devicedetection_cloud/ -e../fiftyone_devicedetection_shared/ -e../fiftyone_devicedetection_examples/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = python setup.py build_clib build_ext {[common]commands} diff --git a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py index c7ea9615..a2511324 100644 --- a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py +++ b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py @@ -35,45 +35,48 @@ # Cloud #pipeline = IpIntelligencePipelineBuilder(resource_key = resource_key).build() -# We create a FlowData object from the pipeline -# this is used to add evidence to and then process +# The pipeline is closed at the end of the with block, which releases the +# native data it holds. +with pipeline: + # We create a FlowData object from the pipeline + # this is used to add evidence to and then process. It is closed at the + # end of its with block, which releases the results of the lookup. + with pipeline.create_flowdata() as flowdata1: -flowdata1 = pipeline.create_flowdata() + # Here we add an IPv4 address as evidence -# Here we add an IPv4 address as evidence + ipv4_address = "185.28.167.78" -ipv4_address = "185.28.167.78" + flowdata1.evidence.add("query.client-ip", ipv4_address) -flowdata1.evidence.add("query.client-ip", ipv4_address) + # Now we process the FlowData -# Now we process the FlowData + flowdata1.process() -flowdata1.process() + # To find the name the IP range is registered under we look at the + # registeredname property inside the IP Intelligence Engine -# To find the name the IP range is registered under we look at the -# registeredname property inside the IP Intelligence Engine + # first we check if this has a meaningful result -# first we check if this has a meaningful result + print("What is IP address " + ipv4_address + " registered as? ") + if flowdata1.ip.registeredname.has_value(): + print(flowdata1.ip.registeredname.value()) + else: + # Output why the value isn't meaningful + print(flowdata1.ip.registeredname.no_value_message()) -print("What is IP address " + ipv4_address + " registered as? ") -if flowdata1.ip.registeredname.has_value(): - print(flowdata1.ip.registeredname.value()) -else: - # Output why the value isn't meaningful - print(flowdata1.ip.registeredname.no_value_message()) + # Now we do the same with a new IP address, this time an IPv6 one -# Now we do the same with a new IP address, this time an IPv6 one + with pipeline.create_flowdata() as flowdata2: -flowdata2 = pipeline.create_flowdata() + ipv6_address = "2001:4860:4860::8888" -ipv6_address = "2001:4860:4860::8888" + flowdata2.evidence.add("query.client-ip", ipv6_address) -flowdata2.evidence.add("query.client-ip", ipv6_address) + flowdata2.process() -flowdata2.process() - -print("What is IP address " + ipv6_address + " registered as? ") -if flowdata2.ip.registeredname.has_value(): - print(flowdata2.ip.registeredname.value()) -else: - print(flowdata2.ip.registeredname.no_value_message()) + print("What is IP address " + ipv6_address + " registered as? ") + if flowdata2.ip.registeredname.has_value(): + print(flowdata2.ip.registeredname.value()) + else: + print(flowdata2.ip.registeredname.no_value_message()) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py index 2e1def33..cc127a01 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/setup.py @@ -54,6 +54,8 @@ def read(file_name): install_requires=[ "fiftyone_ipintelligence", "fiftyone_ipintelligence_translation", + # The first version with close(), which the examples use. + "fiftyone_pipeline_core>=4.5.42", "flask", "ruamel.yaml" ], diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index 062e24da..e418f786 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -96,65 +96,70 @@ def run(self, data_file, truth_file, logger, output, record_limit=RECORD_LIMIT): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) - - writer = csv.DictWriter(output, fieldnames=OUTPUT_FIELDS) - writer.writeheader() - - processed = 0 - found = 0 - with open(truth_file, "r", newline="", encoding="utf-8") as fin: - for truth in csv.DictReader(fin): - if processed >= record_limit: - break - processed += 1 - result = self.compare(pipeline, truth) - if result["Found"]: - found += 1 - writer.writerow(result) - - logger.log("info", - f"Compared {processed} records; {found} resolved to a location") + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) + + writer = csv.DictWriter(output, fieldnames=OUTPUT_FIELDS) + writer.writeheader() + + processed = 0 + found = 0 + with open(truth_file, "r", newline="", encoding="utf-8") as fin: + for truth in csv.DictReader(fin): + if processed >= record_limit: + break + processed += 1 + result = self.compare(pipeline, truth) + if result["Found"]: + found += 1 + writer.writerow(result) + + logger.log("info", + f"Compared {processed} records; {found} resolved to a location") def compare(self, pipeline, truth): - data = pipeline.create_flowdata() - data.evidence.add_from_dict({"query.client-ip": truth["Ip"]}) - data.process() - ip_data = data.ip - - row = { - "DateTimeUtc": truth.get("DateTimeUtc", ""), - "Ip": truth.get("Ip", ""), - "AddressFamily": truth.get("AddressFamily", ""), - "TruthLatitude": truth.get("Latitude", ""), - "TruthLongitude": truth.get("Longitude", ""), - "TruthCountry": truth.get("Country", ""), - "ResultLatitude": "", - "ResultLongitude": "", - "ResultCountry": ExampleUtils.get_human_readable(ip_data, "country"), - "Confidence": ExampleUtils.get_human_readable(ip_data, "locationconfidence"), - "Found": False, - "DistanceKms": "", - } - - latitude = ExampleUtils.get_value(ip_data, "latitude") - longitude = ExampleUtils.get_value(ip_data, "longitude") - if latitude is not None and latitude.has_value() and \ - longitude is not None and longitude.has_value(): - row["ResultLatitude"] = latitude.value() - row["ResultLongitude"] = longitude.value() - row["Found"] = True - try: - row["DistanceKms"] = round(haversine_km( - float(truth["Latitude"]), float(truth["Longitude"]), - float(latitude.value()), float(longitude.value())), 3) - except (ValueError, TypeError): - # The truth record did not provide a usable latitude or - # longitude, so the distance cannot be calculated. - row["DistanceKms"] = "" - - return row + # The FlowData is closed when the method returns, which releases the + # results of the lookup straight away. + with pipeline.create_flowdata() as data: + data.evidence.add_from_dict({"query.client-ip": truth["Ip"]}) + data.process() + ip_data = data.ip + + row = { + "DateTimeUtc": truth.get("DateTimeUtc", ""), + "Ip": truth.get("Ip", ""), + "AddressFamily": truth.get("AddressFamily", ""), + "TruthLatitude": truth.get("Latitude", ""), + "TruthLongitude": truth.get("Longitude", ""), + "TruthCountry": truth.get("Country", ""), + "ResultLatitude": "", + "ResultLongitude": "", + "ResultCountry": ExampleUtils.get_human_readable(ip_data, "country"), + "Confidence": ExampleUtils.get_human_readable(ip_data, "locationconfidence"), + "Found": False, + "DistanceKms": "", + } + + latitude = ExampleUtils.get_value(ip_data, "latitude") + longitude = ExampleUtils.get_value(ip_data, "longitude") + if latitude is not None and latitude.has_value() and \ + longitude is not None and longitude.has_value(): + row["ResultLatitude"] = latitude.value() + row["ResultLongitude"] = longitude.value() + row["Found"] = True + try: + row["DistanceKms"] = round(haversine_km( + float(truth["Latitude"]), float(truth["Longitude"]), + float(latitude.value()), float(longitude.value())), 3) + except (ValueError, TypeError): + # The truth record did not provide a usable latitude or + # longitude, so the distance cannot be calculated. + row["DistanceKms"] = "" + + return row def main(argv): diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index 6d26ce73..eb239813 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -71,42 +71,47 @@ def run(self, data_file, logger, output): add_country_translation(builder) pipeline = builder.add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) - for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + for values in EVIDENCE_VALUES: + self.analyseEvidence(values, pipeline, output) def analyseEvidence(self, evidence, pipeline, output): - data = pipeline.create_flowdata() - - message = [] - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") - if TRANSLATION: - message.append(f"\tquery.translation: {TRANSLATION}\n") - output("".join(message)) - - # Add the IP address evidence, and the language to translate to. - data.evidence.add_from_dict(evidence) - if TRANSLATION: - data.evidence.add("query.translation", TRANSLATION) - - data.process() - - translated = data.get("countrynamestranslated") - - message = [] - message.append("Results:\n") - self.outputList( - translated, "CountryCodesGeographicalAll", - "Geographical countries (most probable first)", message) - self.outputList( - translated, "CountryCodesPopulationAll", - "Population countries (most probable first)", message) - self.outputDropdown(translated, message) - output("".join(message)) + # The FlowData is closed at the end of the with block, which releases + # the results of the lookup straight away. + with pipeline.create_flowdata() as data: + + message = [] + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") + if TRANSLATION: + message.append(f"\tquery.translation: {TRANSLATION}\n") + output("".join(message)) + + # Add the IP address evidence, and the language to translate to. + data.evidence.add_from_dict(evidence) + if TRANSLATION: + data.evidence.add("query.translation", TRANSLATION) + + data.process() + + translated = data.get("countrynamestranslated") + + message = [] + message.append("Results:\n") + self.outputList( + translated, "CountryCodesGeographicalAll", + "Geographical countries (most probable first)", message) + self.outputList( + translated, "CountryCodesPopulationAll", + "Population countries (most probable first)", message) + self.outputDropdown(translated, message) + output("".join(message)) def outputList(self, translated, property_name, label, message): value = translated.get(property_name) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 1236a3b1..37200afd 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -155,7 +155,7 @@ def run(self, data_file, license_key, interactive, logger, output): update_service.on_complete(lambda status, file: update_event.set(status)) # Build the pipeline, configuring it for automatic updates. - IpIntelligencePipelineBuilder( + pipeline = IpIntelligencePipelineBuilder( # The file need not exist when using update on start-up, but its # directory must. Any file present is overwritten, so the engine # is configured to use a temporary copy. @@ -175,35 +175,39 @@ def run(self, data_file, license_key, interactive, logger, output): data_update_product_type = DATA_UPDATE_PRODUCT_TYPE, usage_sharing = False).add_logger(logger).build() - # Block until the start-up update check is complete. - update_event.wait() - output(f"Update on start-up complete - status - {update_event.status}") - - if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: - output("Modifying downloaded file to trigger reload - please wait " - "for that to complete") - update_event.clear() - - # Touching the file metadata triggers the filesystem watcher, - # demonstrating reload when a new file is dropped into place. - now = datetime.now().timestamp() - try: - os.utime(data_file, (now, now)) - except OSError: - raise Exception("Could not modify file time, abandoning example") - - if update_event.wait(120): - output(f"Update on file modification complete, status: " - f"{update_event.status}") + # The pipeline is closed at the end of the with block, which stops the + # automatic updates and the file system watcher, and releases the + # native data it holds. + with pipeline: + # Block until the start-up update check is complete. + update_event.wait() + output(f"Update on start-up complete - status - {update_event.status}") + + if update_event.status == UpdateStatus.AUTO_UPDATE_SUCCESS: + output("Modifying downloaded file to trigger reload - please wait " + "for that to complete") + update_event.clear() + + # Touching the file metadata triggers the filesystem watcher, + # demonstrating reload when a new file is dropped into place. + now = datetime.now().timestamp() + try: + os.utime(data_file, (now, now)) + except OSError: + raise Exception("Could not modify file time, abandoning example") + + if update_event.wait(120): + output(f"Update on file modification complete, status: " + f"{update_event.status}") + else: + output("Update on file modification timed out") else: - output("Update on file modification timed out") - else: - logger.log("error", "Auto update was not successful, abandoning example") - error_message = f"Auto update failed: {update_event.status}" - if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: - output(error_message) - else: - raise Exception(error_message) + logger.log("error", "Auto update was not successful, abandoning example") + error_message = f"Auto update failed: {update_event.status}" + if update_event.status == UpdateStatus.AUTO_UPDATE_ERR_429_TOO_MANY_ATTEMPTS: + output(error_message) + else: + raise Exception(error_message) output("Finished Example") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 8e82567b..4988da7d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -91,11 +91,16 @@ def run(self, data_file, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. An application normally builds + # one pipeline when it starts, uses it for every lookup, and closes + # it when it stops. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) - # carry out some sample lookups - for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + # carry out some sample lookups + for values in EVIDENCE_VALUES: + self.analyseEvidence(values, pipeline, output) def analyseEvidence(self, evidence, pipeline, output): @@ -104,38 +109,41 @@ def analyseEvidence(self, evidence, pipeline, output): # through the pipeline. # Information required is called "evidence", in this case an # IP address supplied under the "query.client-ip" key. - data = pipeline.create_flowdata() + # Closing the FlowData, which the with statement does at the end of + # the block, releases the results of the lookup straight away, + # rather than when the garbage collector next runs. + with pipeline.create_flowdata() as data: - message = [] + message = [] - # List the evidence - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") + # List the evidence + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") - output("".join(message)) + output("".join(message)) - # Add the evidence values to the flow data - data.evidence.add_from_dict(evidence) + # Add the evidence values to the flow data + data.evidence.add_from_dict(evidence) - # Process the flow data. - data.process() + # Process the flow data. + data.process() - message = [] - message.append("Results:\n") + message = [] + message.append("Results:\n") - # Now that it's been processed, the flow data will have - # been populated with the result. In this case, we want - # information about the IP address, which we can get by - # asking for a result matching the key "ip" - ip_data = data.ip + # Now that it's been processed, the flow data will have + # been populated with the result. In this case, we want + # information about the IP address, which we can get by + # asking for a result matching the key "ip" + ip_data = data.ip - # Display the results of the lookup. See the property - # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence - # for details of all available properties. - for name in OUTPUT_PROPERTIES: - self.outputValue(ip_data, name, message) - output("".join(message)) + # Display the results of the lookup. See the property + # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence + # for details of all available properties. + for name in OUTPUT_PROPERTIES: + self.outputValue(ip_data, name, message) + output("".join(message)) def outputValue(self, ip_data, name, message): # Individual result values have a wrapper called diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index debc4a1b..56ea9799 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -88,44 +88,46 @@ def country_dropdown(translated): @app.route("/") def index(): - # Create the flowdata object for this request. - flowdata = pipeline.create_flowdata() - - # Use the IP address supplied as a query parameter, or fall back - # to the address of the requester. - client_ip = request.args.get("client-ip", request.remote_addr) - flowdata.evidence.add("query.client-ip", client_ip) - - # Supply the browser language so the country names are localized. The - # Accept-Language header is used by default, overridable with a - # "translation" query parameter. - accept_language = request.headers.get("Accept-Language") - if accept_language: - flowdata.evidence.add("header.accept-language", accept_language) - translation = request.args.get("translation") - if translation: - flowdata.evidence.add("query.translation", translation) - - # Process the flowdata - flowdata.process() - - ip_data = flowdata.ip - translated = flowdata.get("countrynamestranslated") - - return f""" -

    51Degrees IP Intelligence - Getting Started Web (On-Premise)

    -

    IP address: {client_ip}

    -
      -
    • Registered Name: {get_value(ip_data, "registeredname")}
      • -
    • Registered Owner: {get_value(ip_data, "registeredowner")}
      • -
    • Registered Country: {get_value(ip_data, "registeredcountry")}
      • -
    -

    Country

    -

    Most probable country first, then every other country alphabetically.

    - {country_dropdown(translated)} -

    Add ?client-ip=x.x.x.x to the url to lookup a different address, and - &translation=fr_FR to choose a language.

    - """ + # Create the flowdata object for this request. It is closed when the + # route returns, once the page has been built from it, which releases + # the results of the lookup straight away. + with pipeline.create_flowdata() as flowdata: + + # Use the IP address supplied as a query parameter, or fall back + # to the address of the requester. + client_ip = request.args.get("client-ip", request.remote_addr) + flowdata.evidence.add("query.client-ip", client_ip) + + # Supply the browser language so the country names are localized. The + # Accept-Language header is used by default, overridable with a + # "translation" query parameter. + accept_language = request.headers.get("Accept-Language") + if accept_language: + flowdata.evidence.add("header.accept-language", accept_language) + translation = request.args.get("translation") + if translation: + flowdata.evidence.add("query.translation", translation) + + # Process the flowdata + flowdata.process() + + ip_data = flowdata.ip + translated = flowdata.get("countrynamestranslated") + + return f""" +

    51Degrees IP Intelligence - Getting Started Web (On-Premise)

    +

    IP address: {client_ip}

    +
      +
    • Registered Name: {get_value(ip_data, "registeredname")}
      • +
    • Registered Owner: {get_value(ip_data, "registeredowner")}
      • +
    • Registered Country: {get_value(ip_data, "registeredcountry")}
      • +
    +

    Country

    +

    Most probable country first, then every other country alphabetically.

    + {country_dropdown(translated)} +

    Add ?client-ip=x.x.x.x to the url to lookup a different address, and + &translation=fr_FR to choose a language.

    + """ def main(data_file = None): @@ -138,8 +140,10 @@ def main(data_file = None): "`git submodule update --init --recursive`, and run this " "example again to download the file, or supply the path to a " "data file as the first command line argument.") - build_pipeline(data_file) - app.run(debug=False) + # The pipeline lives as long as the app, and is closed when the server + # stops, which releases the native data it holds. + with build_pipeline(data_file): + app.run(debug=False) if __name__ == "__main__": diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index ec2be8a1..3819776c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -47,14 +47,17 @@ def run(self, data_file, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - engine = pipeline.flow_elements_list["ip"] + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + engine = pipeline.flow_elements_list["ip"] - output("IP Intelligence properties:") - for property_metadata in engine.properties.values(): - output(f" {property_metadata['name']} ({property_metadata['type']}) - " - f"category: {property_metadata['category']}") - if property_metadata.get("description"): - output(f" {property_metadata['description']}") + output("IP Intelligence properties:") + for property_metadata in engine.properties.values(): + output(f" {property_metadata['name']} ({property_metadata['type']}) - " + f"category: {property_metadata['category']}") + if property_metadata.get("description"): + output(f" {property_metadata['description']}") def main(argv): diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 07839ab1..2f4f5e18 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -140,38 +140,43 @@ def run(self, device_file, ip_file, logger, output): .add_logger(logger) \ .build() - for evidence in COMBINED_EVIDENCE: - self.analyse_evidence(evidence, pipeline, output) + # The pipeline is closed at the end of the with block, which closes + # both engines and releases the native data they hold. + with pipeline: + for evidence in COMBINED_EVIDENCE: + self.analyse_evidence(evidence, pipeline, output) def analyse_evidence(self, evidence, pipeline, output): - data = pipeline.create_flowdata() + # The FlowData is closed at the end of the with block, which releases + # the results of both engines straight away. + with pipeline.create_flowdata() as data: - message = [] - message.append("=" * 70 + "\n") - message.append("Input values:\n") - for key in evidence: - message.append(f"\t{key}: {evidence[key]}\n") - output("".join(message)) + message = [] + message.append("=" * 70 + "\n") + message.append("Input values:\n") + for key in evidence: + message.append(f"\t{key}: {evidence[key]}\n") + output("".join(message)) - # Add the evidence values to the flow data and process it. Both - # engines run against the same flow data. - data.evidence.add_from_dict(evidence) - data.process() + # Add the evidence values to the flow data and process it. Both + # engines run against the same flow data. + data.evidence.add_from_dict(evidence) + data.process() - message = [] + message = [] - # Device detection results, keyed under "device". - message.append("Device Detection Results:\n") - for name in DEVICE_PROPERTIES: - self.outputValue(data.device, name, message) + # Device detection results, keyed under "device". + message.append("Device Detection Results:\n") + for name in DEVICE_PROPERTIES: + self.outputValue(data.device, name, message) - # IP intelligence results, keyed under "ip". - message.append("\nIP Intelligence Results:\n") - for name in IP_PROPERTIES: - self.outputValue(data.ip, name, message) + # IP intelligence results, keyed under "ip". + message.append("\nIP Intelligence Results:\n") + for name in IP_PROPERTIES: + self.outputValue(data.ip, name, message) - output("".join(message)) + output("".join(message)) def outputValue(self, aspect_data, name, message): # `get_value` returns None when the property is not present in the diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index 9c7109cc..4785249e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -65,41 +65,47 @@ def run(self, data_file, evidence_yaml, logger, output): auto_update = False, licence_keys = "").add_logger(logger).build() - ExampleUtils.check_data_file(pipeline, logger) - - records = 0 - with open(evidence_yaml, "r") as fin: - for evidence in yaml.load_all(fin): - if records >= RECORD_LIMIT: - break - - # write the yaml document separator - output("---") - self.analyse_evidence(evidence, pipeline, output, yaml) - records += 1 - - # write the yaml document end marker - output("...") - logger.log("info", f"Processed {records} evidence records") + # The pipeline is closed at the end of the with block, which + # releases the native data it holds. + with pipeline: + ExampleUtils.check_data_file(pipeline, logger) + + records = 0 + with open(evidence_yaml, "r") as fin: + for evidence in yaml.load_all(fin): + if records >= RECORD_LIMIT: + break + + # write the yaml document separator + output("---") + self.analyse_evidence(evidence, pipeline, output, yaml) + records += 1 + + # write the yaml document end marker + output("...") + logger.log("info", f"Processed {records} evidence records") def analyse_evidence(self, evidence, pipeline, output, yaml): - data = pipeline.create_flowdata() + # The FlowData is closed at the end of the with block, once the + # values needed have been read, which releases the results of the + # lookup straight away. + with pipeline.create_flowdata() as data: - # Add the evidence values to the flow data. The YAML loader - # returns a dict subclass so convert to a plain dict first. - data.evidence.add_from_dict(dict(evidence)) + # Add the evidence values to the flow data. The YAML loader + # returns a dict subclass so convert to a plain dict first. + data.evidence.add_from_dict(dict(evidence)) - # Process the flow data. - data.process() + # Process the flow data. + data.process() - ip_data = data.ip + ip_data = data.ip - result = dict(evidence) + result = dict(evidence) - # Add the values that we want to store against the record. - result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") - result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") + # Add the values that we want to store against the record. + result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") + result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") buffer = io.BytesIO() yaml.dump(result, buffer) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 310bd727..1d910d81 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -84,11 +84,13 @@ def _build_pipeline(data_file, performance_profile, logger=None): def _process_record(pipeline, record): - data = pipeline.create_flowdata() - data.evidence.add_from_dict(record) - data.process() - value = ExampleUtils.get_value(data.ip, PERFORMANCE_PROPERTY) - return value is not None and value.has_value() + # The FlowData is closed when the function returns, which releases the + # results of the lookup straight away. + with pipeline.create_flowdata() as data: + data.evidence.add_from_dict(record) + data.process() + value = ExampleUtils.get_value(data.ip, PERFORMANCE_PROPERTY) + return value is not None and value.has_value() def _time_lookups(pipeline, records): @@ -115,8 +117,10 @@ def _worker(data_file, performance_profile, records, queue): count and elapsed seconds back to the parent. """ - pipeline = _build_pipeline(data_file, performance_profile) - elapsed = _time_lookups(pipeline, records) + # The pipeline is closed at the end of the with block, which releases + # the native data it holds. + with _build_pipeline(data_file, performance_profile) as pipeline: + elapsed = _time_lookups(pipeline, records) queue.put((len(records), elapsed)) @@ -156,9 +160,10 @@ def run(self, data_file, evidence, logger, output, workers=None, # wall time of the run. elapsed = max(elapsed for _records, elapsed in results) else: - pipeline = _build_pipeline(data_file, performance_profile, logger) - ExampleUtils.check_data_file(pipeline, logger) - elapsed = _time_lookups(pipeline, evidence) + with _build_pipeline( + data_file, performance_profile, logger) as pipeline: + ExampleUtils.check_data_file(pipeline, logger) + elapsed = _time_lookups(pipeline, evidence) per_second = count / elapsed if elapsed > 0 else 0 ms_per_lookup = (elapsed / count) * 1000 if count > 0 else 0 diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index a2d17c30..0ada1f94 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -70,7 +70,9 @@ def test_onpremise_datafile_loads(self): usage_sharing=False, auto_update=False, licence_keys="").build() + self.addCleanup(pipeline.close) data = pipeline.create_flowdata() + self.addCleanup(data.close) data.evidence.add_from_dict({"query.client-ip": "8.8.8.8"}) data.process() # The supplied IP address is echoed back in the results. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index 0008718b..3bc4fbd3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -24,6 +24,13 @@ deps = # these packages is on a package index yet, so it comes from the # working copy as the others do. -e../fiftyone_ipintelligence_translation/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ + -e../../pipeline/fiftyone_pipeline_translation/ # --cov is needed even though coverage of the examples is not the point: # common-ci's run-integration-tests.ps1 moves the .coverage file every # package writes, and throws when there is none, after the tests pass. diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py index b15a227a..2d850cf7 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py @@ -277,5 +277,12 @@ def stage_native_source(): py_modules=["fiftyone_ipintelligence_onpremise"], packages=["fiftyone_ipintelligence_onpremise"], package_dir={"": "src"}, - install_requires=["fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone"], + install_requires=[ + "fiftyone_ipintelligence_shared", + "fiftyone_pipeline_engines_fiftyone", + # The first versions with close(), which releases the native memory + # this package holds. + "fiftyone_pipeline_core>=4.5.42", + "fiftyone_pipeline_engines>=4.5.42", + ], ) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py index 3e129185..deea6331 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/constants.py @@ -29,3 +29,10 @@ 'on-premise IP intelligence engine. Its results ' 'read from native memory, which is not safe to ' 'share between requests.') + +ENGINE_CLOSED = ('The on-premise IP intelligence engine has been closed, ' + 'so it cannot process evidence. Build a new pipeline.') + +RESULTS_CLOSED = ('These IP intelligence results have been closed, so ' + 'their properties cannot be read. Read the values needed ' + 'before closing the FlowData.') diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index c724d64d..37b45904 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -328,6 +328,20 @@ def on_registration(self, pipeline): super(IpIntelligenceOnPremise, self).on_registration(pipeline) self.init_engine() + def close(self): + + """! + + Stop the data file updates and release the native engine. Its data + is freed once no results read from it are still held, so close + every FlowData the engine processed as well. Closing the pipeline + closes the engine. + + """ + + super(IpIntelligenceOnPremise, self).close() + self.engine = None + def get_client_ips(self, flow_data): """! @@ -379,6 +393,9 @@ def process_internal(self, flow_data): """ + if getattr(self, "engine", None) is None: + raise Exception(ENGINE_CLOSED) + evidence = EvidenceIpiSwig() for key, value in flow_data.evidence.get_all().items(): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index ea285c84..40413cf3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -25,6 +25,8 @@ from fiftyone_pipeline_engines.missingproperty_service import MissingPropertyService +from .constants import RESULTS_CLOSED + # Number of decimal places used when converting WKB geometry # properties (e.g. Areas) to their WKT string form. WKT_DECIMAL_PLACES = 16 @@ -71,6 +73,33 @@ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): self.missing_property_service = OnPremiseMissingPropertyService() + def close(self): + + """! + Release the native results, which keep the engine's data in memory + while they are held, even after the engine is closed. Values already + read are unaffected, and reading a property afterwards raises an + exception. Closing the FlowData closes its results. + """ + + self.swig_results = None + + def get(self, key): + + """! + Get a property value, once the results are known to be open + @type key: string + @param key: The key of the property to retrieve + @returns mixed + """ + + # Checked here, because AspectData.get turns an exception from + # get_internal into a missing property message. + if self.swig_results is None: + raise Exception(RESULTS_CLOSED) + + return super(SwigData, self).get(key) + @staticmethod def utf8_string_to_str(utf8_string): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py new file mode 100644 index 00000000..fc53a6d0 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Fails a test that leaves the native results or engine of an on-premise +# engine alive, because they hold the engine's data in memory. The same +# file is in each test folder that runs on-premise engines. +# +# Reference counting frees a native object as soon as nothing holds it. A +# FlowData refers to itself through its evidence, and a pipeline through +# its elements, so when they are not closed only the cycle collector frees +# what they hold. The collector is off while each test runs, so whatever +# the test leaves open is still alive when it ends. Objects that existed +# before the test, such as a pipeline built in setUpClass, are checked once +# every test has run. + +import gc +import importlib + +import pytest + +# The SWIG modules of the on-premise engines, whichever are installed. +NATIVE_MODULES = [ + "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule", + "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule", +] + + +def _native_types(): + types = [] + for name in NATIVE_MODULES: + try: + module = importlib.import_module(name) + except ImportError: + continue + types.append(module.ResultsBaseSwig) + types.append(module.EngineBaseSwig) + return tuple(types) + + +NATIVE_TYPES = _native_types() + +# A failed test is not reported again, because its traceback can hold the +# objects it would have closed. +_failed_tests = set() + +# Objects created when the tests were collected are checked at the end only +# when every test ran, because the teardown that closes them runs with the +# tests. +_some_tests_deselected = False + + +def _alive(): + # The real type is used, because isinstance also asks the object for + # its __class__, which some proxies cannot answer. + return [o for o in gc.get_objects() + if issubclass(type(o), NATIVE_TYPES)] + + +def _describe(names): + return (f"{len(names)} native object(s) still alive " + f"({', '.join(sorted(names))}). Close every FlowData and " + "pipeline the test creates, for example with a with " + "statement, so their native data is released straight away.") + + +def pytest_runtest_logreport(report): + if report.failed: + _failed_tests.add(report.nodeid) + + +def pytest_deselected(items): + global _some_tests_deselected + _some_tests_deselected = True + + +@pytest.fixture(autouse=True) +def native_objects_released(request): + gc.collect() + before = {id(o) for o in _alive()} + gc.disable() + try: + yield + left = [type(o).__name__ for o in _alive() if id(o) not in before] + finally: + gc.enable() + gc.collect() + if left and request.node.nodeid not in _failed_tests: + pytest.fail(_describe(left), pytrace=False) + + +def pytest_sessionfinish(session, exitstatus): + gc.collect() + left = [type(o).__name__ for o in _alive()] + if (left and exitstatus == pytest.ExitCode.OK and + not _some_tests_deselected): + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line( + "After the last test, " + _describe(left), red=True) + session.exitstatus = pytest.ExitCode.TESTS_FAILED diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py index df9f835e..7b14231c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_ipintelligence.py @@ -49,8 +49,13 @@ def setUpClass(cls): .add(cls.engine) \ .build() + @classmethod + def tearDownClass(cls): + cls.pipeline.close() + def process(self, key, value): flowdata = self.pipeline.create_flowdata() + self.addCleanup(flowdata.close) flowdata.evidence.add(key, value) flowdata.process() return flowdata @@ -133,6 +138,24 @@ def test_process_ipv6(self): ipv6 = flowdata.ip.get("ipv6") self.assertTrue(ipv6.has_value()) + def test_flowdata_close_releases_results(self): + """! + Closing the FlowData releases its native results, and reading a + property afterwards raises an exception. + """ + + with self.pipeline.create_flowdata() as flowdata: + flowdata.evidence.add( + "query.client-ip", TestConstants.IPV4_ADDRESS) + flowdata.process() + registered_name = flowdata.ip.get("registeredname") + + self.assertTrue(registered_name.has_value()) + self.assertIsNone(flowdata.ip.swig_results) + with self.assertRaises(Exception) as context: + flowdata.ip.get("registeredname") + self.assertIn("closed", str(context.exception)) + def test_missing_property(self): """! Accessing a property that does not exist raises an exception @@ -208,10 +231,12 @@ def test_asn_data_file_opens(self): data_file_path=TestConstants.get_asn_data_file_path(), licence_keys="") pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) self.assertIn("asnname", engine.properties) flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) flowdata.evidence.add("query.client-ip", TestConstants.IPV4_ADDRESS) flowdata.process() @@ -262,3 +287,31 @@ def test_pipeline_builder_refuses_cache(self): cache=LRUEngineCache()).build() self.assertIn("results cache", str(context.exception)) + + +@unittest.skipUnless( + os.path.isfile(data_file), + "No IP intelligence data file found at '%s'. Run ci/fetch-assets.ps1 " + "or set the IPINTELLIGENCEDATAFILE environment variable." % data_file) +class IpIntelligenceCloseTests(unittest.TestCase): + + def test_pipeline_close_releases_engine(self): + """! + Closing the pipeline releases the native engine, and processing + afterwards raises an exception. + """ + + engine = IpIntelligenceOnPremise( + data_file_path=data_file, + licence_keys="") + pipeline = PipelineBuilder().add(engine).build() + + pipeline.close() + + self.assertIsNone(engine.engine) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("query.client-ip", TestConstants.IPV4_ADDRESS) + with self.assertRaises(Exception) as context: + flowdata.process() + self.assertIn("closed", str(context.exception)) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini index 3dc3b4c9..211e4571 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tox.ini @@ -24,6 +24,12 @@ deps = # of it in beside the wheel tox builds and made examples and on-premise # depend on each other. -e../fiftyone_ipintelligence_shared/ + # The pipeline packages come from this checkout rather than a package + # index, so the tests run with the pipeline code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = python setup.py build_clib build_ext {[common]commands} diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index daa6b387..c6f83c7c 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -94,6 +94,34 @@ def process(self): return self + def close(self): + + """! + Release the native resources held by the element data in this + FlowData, such as the results of an on-premise engine, which keep + the engine's data in memory until they are released. Use a with + statement so that this happens as soon as the FlowData is finished + with, rather than when the garbage collector next runs. + + Only element data with a close method of its own is closed, so + element data that needs no cleanup, such as an entry in a results + cache, is left alone. Values already read are unaffected. + + """ + + for element_data in list(self.data.values()): + # Looked up on the class, because ElementData treats an + # attribute it does not have as the name of a property. + close = getattr(type(element_data), "close", None) + if callable(close): + close(element_data) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def get_from_element(self, flow_element): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py index 87ee2800..73e6922d 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowelement.py @@ -77,6 +77,17 @@ def on_registration(self, pipeline): pass + def close(self): + + """! + Release the resources the element holds. The pipeline calls this + when it is closed. This element holds none, and elements that hold + native data or threads override it. + + """ + + pass + def get_evidence_key_filter(self): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py index cac6aea1..4ee68108 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py @@ -33,7 +33,7 @@ class Pipeline: """ - def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = False, data_file_update_service = None): + def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = False, data_file_update_service = None, auto_close_elements = True): """! Pipeline constructor. @@ -49,6 +49,12 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = exceptions added to FlowData.errors otherwise will throw the exception occurred during the processing of first element. + @type auto_close_elements: boolean + @param auto_close_elements: If true, the default, closing the + pipeline closes its flow elements. Set it to false when the elements + are shared with another pipeline, and close each element once every + pipeline using it is finished with. + @rtype: Pipeline @return: Returns a Pipeline @@ -63,6 +69,8 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = self.flow_elements_display_list = [] self.suppress_process_exceptions = suppress_process_exceptions + + self.auto_close_elements = auto_close_elements if data_file_update_service != None: self.data_file_update_service = data_file_update_service @@ -92,6 +100,44 @@ def create_flowdata(self): return FlowData(self) + def close(self): + + """! + Close the flow elements, which releases the native data they hold + and stops the updates of their data files, unless the pipeline was + built with auto_close_elements set to false. Use a with statement + so that this happens when the pipeline is finished with. + + FlowData created by the pipeline are not closed by it. Close each + one as well, because the results it holds keep the engine's data in + memory until they are released. + + """ + + if not self.auto_close_elements: + return + + # Every element is closed even if one fails, and the first failure + # is raised once they all have been. + failure = None + for flow_element in self.flow_elements: + close = getattr(flow_element, "close", None) + if close is None: + continue + try: + close() + except Exception as e: + if failure is None: + failure = e + if failure is not None: + raise failure + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + def log(self, level, message): """! diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py index 72f89752..c3fa52e0 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py @@ -52,6 +52,9 @@ def __init__(self, settings= {}): while processing is re-raised from flow_data.process(); if True, errors are stored on flow_data.errors and process() returns normally. Recommended True for web apps so a processing failure degrades gracefully instead of failing the request. + `auto_close_elements (Bool)` - Whether closing the pipeline closes its flow elements + (default true). Set to False when the elements are shared with another pipeline, + and close each element once every pipeline using it is finished with. @return: Returns a Pipeline Builder """ @@ -82,6 +85,11 @@ def __init__(self, settings= {}): else: self.suppress_process_exceptions = False + if "auto_close_elements" in settings: + self.auto_close_elements = settings["auto_close_elements"] + else: + self.auto_close_elements = True + def get_javascript_elements(self): """ Adds the JavaScriptBuilder, JSONBundler and SequenceElement to the pipeline if @@ -148,7 +156,7 @@ def build(self): self.flow_elements.extend(self.get_javascript_elements()) self.flow_elements.extend(self.get_setheader_elements()) - return Pipeline(self.flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service) + return Pipeline(self.flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements) def add_logger(self, logger): """ @@ -176,27 +184,46 @@ def build_from_configuration(self, config): @rtype: Pipeline @return: Returns built pipeline + If an element fails to build, the elements already built are closed + and the error is raised. + """ flow_elements = [] - for element in config["PipelineOptions"]["Elements"]: - flow_element = importlib.import_module(element["elementPath"]).__getattribute__(element["elementName"]) - if("elementParameters" in element): - arguments = self._map_properties_names( - { - "verify_md5": "data_update_verify_md5", - "data_file_update_base_url": "data_update_url" - }, - element["elementParameters"], - ) - - flow_element = flow_element(**arguments) - else: - flow_element = flow_element() - flow_elements.append(flow_element) - - return Pipeline(flow_elements, suppress_process_exceptions=self._get_suppress_process_exceptions(config)) + try: + for element in config["PipelineOptions"]["Elements"]: + flow_element = importlib.import_module(element["elementPath"]).__getattribute__(element["elementName"]) + if("elementParameters" in element): + arguments = self._map_properties_names( + { + "verify_md5": "data_update_verify_md5", + "data_file_update_base_url": "data_update_url" + }, + element["elementParameters"], + ) + + flow_element = flow_element(**arguments) + else: + flow_element = flow_element() + flow_elements.append(flow_element) + + return Pipeline(flow_elements, suppress_process_exceptions=self._get_suppress_process_exceptions(config), auto_close_elements=self.auto_close_elements) + except BaseException: + # A failure to close is logged rather than raised, so the caller + # sees the error that stopped the build. + for flow_element in flow_elements: + close = getattr(flow_element, "close", None) + if close is None: + continue + try: + close() + except Exception as e: + self.logger.log( + "error", + "Could not close {} after the pipeline failed to " + "build: {}".format(type(flow_element).__name__, e)) + raise def _get_suppress_process_exceptions(self, config): """ diff --git a/pipeline/fiftyone_pipeline_core/tests/test_close.py b/pipeline/fiftyone_pipeline_core/tests/test_close.py new file mode 100644 index 00000000..e99583d5 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_close.py @@ -0,0 +1,182 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_core.elementdata import ElementData +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + + +class ClosingData(ElementData): + """Element data with a close method of its own, as the results of an + on-premise engine have.""" + + def __init__(self, flow_element): + super(ClosingData, self).__init__(flow_element) + self.closed = False + + def close(self): + self.closed = True + + +class ClosingElement(FlowElement): + """Adds ClosingData to each FlowData, and counts its own closes.""" + + def __init__(self): + super(ClosingElement, self).__init__() + self.datakey = "closing" + self.closes = 0 + + def process_internal(self, flowdata): + flowdata.set_element_data(ClosingData(self)) + + def close(self): + self.closes += 1 + + +class PlainElement(FlowElement): + """Adds element data that needs no cleanup.""" + + def __init__(self): + super(PlainElement, self).__init__() + self.datakey = "plain" + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, {"close": 1})) + + +class FailingElement(FlowElement): + """Fails to close.""" + + def __init__(self): + super(FailingElement, self).__init__() + self.datakey = "failing" + + def close(self): + raise Exception("close failed") + + +class RecordedElement(FlowElement): + """Records each instance that is closed, for a test whose pipeline + failed to build and so cannot be reached.""" + + closed = [] + + def __init__(self): + super(RecordedElement, self).__init__() + self.datakey = "recorded" + + def close(self): + RecordedElement.closed.append(self) + + +class BrokenElement(FlowElement): + """Fails to build.""" + + def __init__(self): + raise ValueError("element failed to build") + + +def build(*elements, **settings): + settings["add_javascript_builder"] = False + settings["use_setheader_properties"] = False + builder = PipelineBuilder(settings) + for element in elements: + builder.add(element) + return builder.build() + + +class CloseTests(unittest.TestCase): + + # Element data with a close method is closed with the FlowData, and + # element data without one is left alone, even with a property called + # close. + def test_flowdata_closes_element_data(self): + with build(ClosingElement(), PlainElement()) as pipeline: + with pipeline.create_flowdata() as flowdata: + flowdata.process() + closing = flowdata.get("closing") + self.assertFalse(closing.closed) + self.assertTrue(closing.closed) + self.assertEqual(1, flowdata.get("plain").get("close")) + + # The with statement closes the FlowData when an exception is raised. + def test_flowdata_closed_on_exception(self): + with build(ClosingElement()) as pipeline: + with self.assertRaises(ValueError): + with pipeline.create_flowdata() as flowdata: + flowdata.process() + closing = flowdata.get("closing") + raise ValueError() + self.assertTrue(closing.closed) + + # A FlowData that was never processed holds nothing to close. + def test_unprocessed_flowdata_closes(self): + with build(ClosingElement()) as pipeline: + pipeline.create_flowdata().close() + + def test_pipeline_closes_elements(self): + element = ClosingElement() + with build(element): + self.assertEqual(0, element.closes) + self.assertEqual(1, element.closes) + + # Elements shared with another pipeline are left for the caller to + # close. + def test_pipeline_leaves_elements_open(self): + element = ClosingElement() + with build(element, auto_close_elements=False): + pass + self.assertEqual(0, element.closes) + + # Every element is closed even when one fails, and the failure is + # raised afterwards. + def test_pipeline_closes_every_element(self): + element = ClosingElement() + pipeline = build(FailingElement(), element) + with self.assertRaises(Exception) as context: + pipeline.close() + self.assertEqual("close failed", str(context.exception)) + self.assertEqual(1, element.closes) + + # When an element in a configuration fails to build, the elements built + # before it are closed, and the error that stopped the build is raised. + def test_failed_configuration_closes_built_elements(self): + RecordedElement.closed = [] + config = { + "PipelineOptions": { + "Elements": [ + {"elementName": "RecordedElement", "elementPath": __name__}, + {"elementName": "BrokenElement", "elementPath": __name__} + ] + } + } + with self.assertRaises(ValueError) as context: + PipelineBuilder().build_from_configuration(config) + self.assertEqual("element failed to build", str(context.exception)) + self.assertEqual(1, len(RecordedElement.closed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py index 59168577..c4c6b047 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/datafile_update_service.py @@ -90,6 +90,10 @@ def update_data_file(self, data_file): @param data_file: Datafile to update """ + # A data file whose engine has been closed is not updated. + if not self.__is_registered(data_file): + return False + # exit if already updating if data_file.updating: @@ -230,6 +234,9 @@ def check_next_update(self, data_file, run_after_error = False): @param data_file: Datafile to check updates for """ + if not self.__is_registered(data_file): + return + if data_file.update_on_start and not data_file.updated_on_start and not run_after_error: self.update_data_file(data_file) @@ -241,6 +248,7 @@ def check_next_update(self, data_file, run_after_error = False): t = threading.Timer(interval, self.async_update_data_file, args=[data_file]) t.daemon = True t.name = "AutoUpdate - Scheduled" + data_file.update_timer = t t.start() def register_pipeline(self, pipeline): @@ -265,6 +273,26 @@ def register_data_file(self, data_file): data_file.last_changed_time = os.stat(data_file.path).st_mtime self.track_local_data_file_changes(data_file) + def unregister_data_file(self, data_file): + """! + Stop the checks and updates of a datafile, for example when its + engine is closed. Checks already scheduled are cancelled, and a + check already running schedules no other, so nothing is left + holding the data file or its engine. + @type data_file: DataFile + @param data_file: Datafile to stop checking + """ + + data_file.registered = False + for name in ("update_timer", "watcher_timer"): + timer = getattr(data_file, name, None) + if timer is not None: + timer.cancel() + + def __is_registered(self, data_file): + # A data file checked without being registered first is checked as + # before. Only one that has been unregistered is left alone. + return getattr(data_file, "registered", True) def track_local_data_file_changes(self, data_file): """! @@ -272,10 +300,12 @@ def track_local_data_file_changes(self, data_file): the file system watcher. A refresh that fails is logged, and reported with AUTO_UPDATE_REFRESH_FAILED as on the update route. The next check is scheduled whatever happens, so that a later file - is still picked up. + is still picked up, until the data file is unregistered. @type data_file: DataFile @param data_file: Datafile to check updates for """ + if not self.__is_registered(data_file): + return try: changed_time = os.stat(data_file.path).st_mtime except OSError: @@ -296,13 +326,15 @@ def track_local_data_file_changes(self, data_file): "' failed. " + str(e)) self.__emit(UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, data_file) finally: - t = threading.Timer( - self.FILE_SYSTEM_WATCHER_INTERVAL_SECONDS, - self.track_local_data_file_changes, - args=[data_file]) - t.daemon = True - t.name = "AutoUpdate - FileWatcher" - t.start() + if self.__is_registered(data_file): + t = threading.Timer( + self.FILE_SYSTEM_WATCHER_INTERVAL_SECONDS, + self.track_local_data_file_changes, + args=[data_file]) + t.daemon = True + t.name = "AutoUpdate - FileWatcher" + data_file.watcher_timer = t + t.start() def on_complete(self, on_complete_method): self.on_complete_methods.append(on_complete_method) diff --git a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py index f22f654c..5ef3556a 100644 --- a/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py +++ b/pipeline/fiftyone_pipeline_engines/src/fiftyone_pipeline_engines/engine.py @@ -150,6 +150,22 @@ def on_registration(self, pipeline): pipeline.data_file_update_service = DataFileUpdateService(pipeline) pipeline.data_file_update_service.register_data_file(self.data_file) + def close(self): + + """! + + Stop the checks and updates of the engine's data file, whose timers + would otherwise keep the engine alive. Engines that hold native data + release it as well. + + """ + + if self.data_file: + for pipeline in self.pipelines: + service = getattr(pipeline, "data_file_update_service", None) + if service is not None: + service.unregister_data_file(self.data_file) + def register_data_file(self, data_file): """! diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py index af290377..5fd9c872 100644 --- a/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py +++ b/pipeline/fiftyone_pipeline_engines/tests/test_datafile_update_service.py @@ -73,10 +73,16 @@ def __init__(self, pending, function, args): self.pending = pending self.function = function self.args = args + self.cancelled = False def start(self): self.pending.append(self) + def cancel(self): + self.cancelled = True + if self in self.pending: + self.pending.remove(self) + class FileSystemWatcherTests(unittest.TestCase): @@ -149,6 +155,39 @@ def testFileSystemWatcherKeepsWatchingWhileFileIsMissing(self): self.assertNotIn( UpdateStatus.AUTO_UPDATE_REFRESH_FAILED, self.statuses) + # Unregistering, as closing the engine does, cancels the next check, and + # a check already running neither refreshes nor schedules another, so + # nothing is left holding the data file. + def testUnregisterStopsFileSystemWatcher(self): + data_file = StandInDataFile(self.path) + self.service.register_data_file(data_file) + check = self.pending[0] + + self.service.unregister_data_file(data_file) + self.assertTrue(check.cancelled) + self.assertEqual(0, len(self.pending)) + + self.write("version 2") + check.function(*check.args) + self.assertEqual("version 1", data_file.data) + self.assertEqual(0, len(self.pending)) + + # Unregistering cancels the next automatic update, and an update that + # starts afterwards does nothing. + def testUnregisterStopsAutomaticUpdates(self): + data_file = StandInDataFile(self.path) + data_file.file_system_watcher = False + data_file.auto_update = True + data_file.polling_interval = 30 + data_file.update_time_maximum_randomisation = 0 + self.service.register_data_file(data_file) + check = self.pending[0] + + self.service.unregister_data_file(data_file) + self.assertTrue(check.cancelled) + self.assertEqual(0, len(self.pending)) + self.assertFalse(self.service.update_data_file(data_file)) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 55ef341f..3173318e 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -122,6 +122,17 @@ def __init__( def __del__(self): self.thread_pool.shutdown(wait=True) + def close(self): + + """! + Wait for any usage data already being sent, then stop the threads + that send it. The pipeline calls this when it is closed. + + """ + + super(ShareUsage, self).close() + self.thread_pool.shutdown(wait=True) + def get_constant_xml(self): if self.constant_xml is None: coreVersion = version("fiftyone_pipeline_core") From dfa67572d385c799e985d6f9dfbc12413e95e6d3 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Wed, 4 Nov 2026 23:04:16 +0000 Subject: [PATCH 12/26] Look for the resource keys link the cloud engine now gives in the cloud tests (#38) --- .../fiftyone_devicedetection_cloud/tests/test_cloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 3151778e..c5369df1 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -121,7 +121,7 @@ def test_missing_property_service_element_not_found(self): result = str(raised.exception) self.assertIn("notpresent", result) self.assertIn("['device']", result) - self.assertIn("https://51degrees.com/documentation/_info__resource_keys.html", result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_engine_init_performance(self): """! @@ -172,7 +172,7 @@ def test_missing_property_service_not_found_anywhere(self): self.assertIn("notpresent", result) self.assertIn("device", result) self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result) - self.assertIn("https://51degrees.com/documentation/_info__resource_keys.html", result) + self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) def test_cloud_request_origin(self): """! From 8b03b74394e3f9e9e5a6784005b130cd96328c24 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 5 Nov 2026 06:58:21 +0000 Subject: [PATCH 13/26] Allow only http and https in the callback URL, and test the zone and port rules (#40) --- .../fiftyone_pipeline_core/callback_url.py | 54 ++++++++++--- .../javascriptbuilder.py | 18 ++++- .../tests/test_callback_url.py | 77 ++++++++++++++++++- .../tests/test_javascriptbuilder_variables.py | 49 ++++++++++++ 4 files changed, 182 insertions(+), 16 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py index afafa240..b18fbb13 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -24,9 +24,10 @@ The callback URL the client script posts to for updated JSON. The protocol and host usually come from the request, and the script holds the URL in a single-quoted string, so the URL is checked before it is written into the -script. A URL that is not valid is refused. In the user information, in -anything after the host and port, and in the endpoint, the characters a URL -cannot hold or that could end the string or the script are percent-encoded. +script. A URL that is not valid, including one whose protocol is not http or +https, is refused. In the user information, in anything after the host and +port, and in the endpoint, the characters a URL cannot hold or that could end +the string or the script are percent-encoded. See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url """ @@ -58,9 +59,13 @@ r"[\t-\r \u0085\u00a0\u1680\u2000-\u200a" r"\u2028\u2029\u202f\u205f\u3000]*") -# A scheme as RFC 3986 section 3.1 defines one, being a letter followed by -# letters, digits, '+', '-' and '.'. -SCHEME_PATTERN = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*") +# The schemes the script can send its request with. It uses the URL only with +# fetch and XMLHttpRequest, which reach a server with no other scheme. +SCHEMES = ("http", "https") + +# Lower-cases the ASCII letters A to Z and nothing else. +_ASCII_LOWER = str.maketrans( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") # Where the authority ends, which is at the first '/', '?' or '#' of the # host. @@ -105,6 +110,37 @@ def is_blank(text): or BLANK_PATTERN.fullmatch(text) is not None) +def ascii_lower(text): + + """! + The text with the ASCII letters A to Z in lower case and every other + character as it is, so that no letter outside ASCII can stand in for an + ASCII one, as str.casefold would let the long s stand in for "s". + + @type text: str + @param text: The text + @rtype: str + @return: The text with A to Z in lower case + """ + + return text.translate(_ASCII_LOWER) + + +def is_scheme(protocol): + + """! + Whether the protocol is http or https, ignoring the case of its ASCII + letters only. + + @type protocol: str + @param protocol: The protocol + @rtype: bool + @return: True if the protocol is http or https + """ + + return ascii_lower(protocol) in SCHEMES + + def callback_url(protocol, host, endpoint): """! @@ -114,7 +150,7 @@ def callback_url(protocol, host, endpoint): the rest as given, apart from the percent-encoding. @type protocol: str - @param protocol: The protocol, such as https + @param protocol: The protocol, http or https in any case @type host: str @param host: The host, which can start with user information and end with a port, a path, a query or a fragment @@ -134,7 +170,7 @@ def callback_url(protocol, host, endpoint): elif host_has_slash and endpoint_has_slash: endpoint = endpoint[1:] built = protocol + "://" + host + endpoint - if SCHEME_PATTERN.fullmatch(protocol) is None: + if not is_scheme(protocol): return CallbackUrl(False, built) match = AUTHORITY_END_PATTERN.search(host) end = len(host) if match is None else match.start() @@ -143,7 +179,7 @@ def callback_url(protocol, host, endpoint): return CallbackUrl(False, built) return CallbackUrl( True, - protocol.lower() + "://" + authority + ascii_lower(protocol) + "://" + authority + _encoded(host[end:]) + _encoded(endpoint)) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py index 31d063a7..bdb85c50 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py @@ -38,7 +38,7 @@ from .evidence_keyfilter import EvidenceKeyFilter from .elementdata_dictionary import ElementDataDictionary from .constants import Constants -from .callback_url import callback_url +from .callback_url import callback_url, is_scheme # Evidence keys the rendered script is not configured with. Both are @@ -255,9 +255,10 @@ def __init__(self, settings = {} ): * starting with a digit) and not a reserved word. An invalid * configured name raises a ValueError, and an invalid requested name * is ignored with a warning logged. - * @param {string} options.protocol The protocol ("http" or "https") - * used by the client side callback url. - * This can be overriden with header.protocol evidence + * @param {string} options.protocol The protocol of the client side + * callback url, http or https in any case of its ASCII letters. Any + * other configured protocol raises a ValueError. Left out, the + * header.protocol evidence is used, or https when there is none. * @param {string} options.host The host of the client side * callback url. This can be overriden with header.host evidence. * @param {string} options.endpoint The endpoint of the client side @@ -286,6 +287,15 @@ def __init__(self, settings = {} ): "JavaScript identifier (ASCII letters, digits, '_' and '$', " "not starting with a digit) and not a reserved word.") self.settings['_protocol'] = settings["protocol"] if "protocol" in settings else None + + # A protocol given here is used for every request, so one the script + # cannot send its request with is refused now rather than logged on + # the first request. + protocol = self.settings['_protocol'] + if protocol and not (isinstance(protocol, str) and is_scheme(protocol)): + raise ValueError( + "The JavaScript builder's protocol setting must be http or " + "https.") self.settings['_host'] = settings["host"] if "host" in settings else None self.settings['_endpoint'] = settings["endpoint"] if "endpoint" in settings else '' self.settings['_enableCookies'] = settings["enable_cookies"] if "enable_cookies" in settings else True diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py index 9048401d..129fee0d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -137,19 +137,90 @@ def test_accepts_the_host(self, host, expected): def test_refuses_the_host(self, host): self.assertFalse(_url("https", host).valid, repr(host)) + @parameterized.expand([ + ("[fe80::1%25]",), + ("[fe80::1%25'+alert(1)+']",), + ("[fe80::1%eth0]",), + ("[fe80::1%25eth%0]",), + ("[::1]x",), + ("[::1]80",), + ("[::1]:80x",), + ("[::1]]",), + ]) + def test_refuses_the_zone_or_the_text_after_the_bracket(self, host): + """Inside the brackets only an IPv6 address, optionally followed by + '%25' and a zone of unreserved characters, and after them only a + port. The zone check is what keeps a quote out of the script for a + bracketed host.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + def test_accepts_a_zone_followed_by_a_port(self): + self.assertEqual( + _valid("https://[fe80::1%25eth0]:8080/51dpipeline/json"), + _url("https", "[fe80::1%25eth0]:8080")) + + def test_checks_the_port_by_its_value(self): + """A port goes by its value, so leading zeros are allowed, and a + port too long to read is refused rather than read as small enough.""" + + self.assertEqual( + _valid("https://example.com:00080/51dpipeline/json"), + _url("https", "example.com:00080")) + self.assertEqual( + _valid("https://example.com:0065535/51dpipeline/json"), + _url("https", "example.com:0065535")) + for host in [ + "example.com:0065536", + "example.com:123456", + "example.com:4294967296", + "example.com:99999999999999999999"]: + self.assertFalse(_url("https", host).valid, host) + + def test_encodes_the_characters_rule_8_lists(self): + """Each character rule 8 lists, in the user information, after the + host and in the endpoint, and a '%' without two hexadecimal digits + after it, even at the end.""" + + self.assertEqual( + _valid("https://u%22%5E%60%7B%7C%7D@example.com/%22%5E%60%7B%7C%7D" + "/j%22%5E%60%7B%7C%7D"), + callback_url( + "https", 'u"^`{|}@example.com/"^`{|}', '/j"^`{|}')) + self.assertEqual( + _valid("https://example.com/a%25/51dpipeline/json"), + _url("https", "example.com/a%")) + self.assertEqual( + _valid("https://example.com/a%254/51dpipeline/json"), + _url("https", "example.com/a%4")) + self.assertEqual( + _valid("https://example.com/a%41/51dpipeline/json"), + _url("https", "example.com/a%41")) + self.assertEqual( + _valid("https://example.com/json%25"), + callback_url("https", "example.com", "/json%")) + def test_the_invalid_url_is_the_one_built(self): self.assertEqual( CallbackUrl(False, "https://exa mple.com/51dpipeline/json"), _url("https", "exa mple.com")) def test_checks_the_scheme(self): - for protocol in ["https", "http", "ftp", "h+t-p.s"]: + """Only http and https, ignoring the case of ASCII letters only, so + a letter outside ASCII that a case-insensitive comparison can match + to an ASCII one, such as the long s, is refused.""" + + for protocol in ["https", "http", "HtTp"]: self.assertTrue(_url(protocol, "example.com").valid, protocol) self.assertEqual( _valid("https://example.com/51dpipeline/json"), _url("HTTPS", "example.com")) - for protocol in ["1http", "h t", "x');alert(1);//", "a://b"]: - self.assertFalse(_url(protocol, "example.com").valid, protocol) + for protocol in [ + "ftp", "h+t-p.s", "ws", "javascript", "httpx", "https ", + "1http", "h t", "x');alert(1);//", "a://b", "http\u017f", + "HTTP\u017f"]: + self.assertFalse( + _url(protocol, "example.com").valid, repr(protocol)) @parameterized.expand([ ("example.com/');alert(1);//",), diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index 4c9302f4..7b601e61 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -24,6 +24,8 @@ import unittest from unittest import mock +from parameterized import parameterized + from fiftyone_pipeline_core import javascriptbuilder from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder @@ -211,6 +213,33 @@ def test_refused_url_is_logged_on_one_line(self): self.assertNotIn(c, errors[0]) self.assertIn("a\\nb.com\\r\\u2028", errors[0]) + def test_refused_protocol_from_the_request_is_left_out_and_logged_once( + self): + """A protocol from the request that is neither http nor https gives + a script with no callback, and is logged once like any other URL + that is not valid.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json"}}).add_logger(logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + for _ in range(2): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.host", "example.com") + flowdata.evidence.add("header.protocol", "ftp") + flowdata.process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + + for variables in captured.variables: + self.assertNotIn("_url", variables) + self.assertFalse(variables["_updateEnabled"]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'ftp://example.com/json'", errors[0]) + def test_blank_host_gives_no_callback_and_is_not_logged(self): """A host made only of whitespace is not set, so the script has no callback and nothing is logged.""" @@ -232,5 +261,25 @@ def test_host_from_the_request_is_written_as_checked(self): "https://example.com/%27);alert(1);//json", rendered[0]["_url"]) +class JavaScriptBuilderProtocolSettingTests(unittest.TestCase): + + @parameterized.expand([ + ("ftp",), ("ws",), ("javascript",), ("https ",), ("http\u017f",), + ]) + def test_invalid_configured_protocol_is_refused(self, protocol): + """A protocol in the settings is used for every request, so one the + script cannot send its request with is refused when the builder is + created.""" + + with self.assertRaises(ValueError): + javascriptbuilder.JavascriptBuilderElement({"protocol": protocol}) + + @parameterized.expand([("http",), ("https",), ("HTTPS",)]) + def test_valid_configured_protocol_is_accepted(self, protocol): + element = javascriptbuilder.JavascriptBuilderElement( + {"protocol": protocol}) + self.assertEqual(protocol, element.settings["_protocol"]) + + if __name__ == "__main__": unittest.main() From 081386da8930066d81e2c78942f1499335e6cb45 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Thu, 5 Nov 2026 08:29:31 +0000 Subject: [PATCH 14/26] Test every package against the packages it depends on in this checkout (#41) --- ci/packages.Tests.ps1 | 5 +++ ci/packages.ps1 | 45 +++++++++++++++++++ ci/run-unit-tests.ps1 | 10 +++++ .../fiftyone_devicedetection_cloud/tox.ini | 6 +++ .../fiftyone_ipintelligence_cloud/tox.ini | 6 +++ .../fiftyone_ipintelligence_shared/tox.ini | 6 +++ .../tox.ini | 4 ++ location/tox.ini | 5 +++ 8 files changed, 87 insertions(+) diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 77ecd47e..2a2887d9 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -285,6 +285,11 @@ Describe 'the real packages.json' { $ipi.Count | Should -Be 5 $ipi.FirstVersion | Select-Object -Unique | Should -Be '4.5.0' } + + It 'has every tox.ini install the packages its package depends on' { + Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | + Should -BeNullOrEmpty + } } } diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 5043d0aa..70e6af0c 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -30,6 +30,51 @@ function Test-PathUnder([string]$Path, [string]$Folder) { return $Path -eq $Folder -or $Path.StartsWith("$Folder/") } +# Each tox.ini that does not install every package its package depends on, +# directly or through another, from this checkout, as a line naming what is +# missing. Taken from a package index instead, a dependency is tested at its +# last release rather than with the code it is released with. +function Get-ToxGaps([string]$RepoPath, $Packages = $null) { + if ($null -eq $Packages) { + $Packages = Get-Packages $RepoPath + } + $byName = @{} + foreach ($package in $Packages) { + $byName[$package.Name] = $package + } + $gaps = @() + foreach ($package in $Packages) { + $tox = Join-Path $RepoPath $package.Path 'tox.ini' + if ((Test-Path $tox) -eq $false) { + continue + } + $text = Get-Content -Raw $tox + $needed = [Collections.Generic.HashSet[string]]::new() + $queue = [Collections.Generic.Queue[string]]::new() + foreach ($name in $package.DependsOn) { + $queue.Enqueue($name) + } + while ($queue.Count -gt 0) { + $name = $queue.Dequeue() + if ($needed.Add($name) -eq $true) { + foreach ($next in $byName[$name].DependsOn) { + $queue.Enqueue($next) + } + } + } + foreach ($name in $needed) { + $relative = [IO.Path]::GetRelativePath( + (Join-Path $RepoPath $package.Path), + (Join-Path $RepoPath $byName[$name].Path)).Replace('\', '/') + $pattern = '(?m)^\s*-e\s*' + [regex]::Escape($relative) + '/?\s*$' + if ($text -notmatch $pattern) { + $gaps += "$($package.Path)/tox.ini does not install $name from $relative" + } + } + } + return $gaps +} + # The names of the given packages and of every package that depends on one of # them, directly or through another. function Get-DependentClosure($Packages, [string[]]$Names) { diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index 77ad8cad..e74a60ff 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -3,5 +3,15 @@ param ( ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') +. (Join-Path $PSScriptRoot 'packages.ps1') + +# Each package's tests take the packages it depends on from this checkout, so +# they test the code those packages are released with. A tox.ini that takes +# one from PyPI instead stops the tests before they run. +$gaps = @(Get-ToxGaps (Split-Path $PSScriptRoot)) +if ($gaps.Count -gt 0) { + throw "A tox.ini takes a package from PyPI that this checkout has:`n" + + ($gaps -join "`n") +} Invoke-ProductStep 'run-unit-tests' $RepoName $PSBoundParameters diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index 14fad039..a0d62c6b 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -12,7 +12,13 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. -e../fiftyone_devicedetection_shared/ + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_devicedetection_cloud {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini index 4aa63411..668932c9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_cloud/tox.ini @@ -12,7 +12,13 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. -e../fiftyone_ipintelligence_shared/ + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_cloud {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini index 2bf335a2..98972fd0 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_shared/tox.ini @@ -12,6 +12,12 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_engines/ + -e../../pipeline/fiftyone_pipeline_engines_fiftyone/ + -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_shared {tty:--color=yes} {posargs} pass_env = diff --git a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini index 11283b9e..b055ee03 100644 --- a/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_translation/tox.ini @@ -12,6 +12,10 @@ package = wheel wheel_build_env = .pkg deps = {[common]deps} + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../../pipeline/fiftyone_pipeline_core/ + -e../../pipeline/fiftyone_pipeline_translation/ commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_translation {tty:--color=yes} {posargs} pass_env = diff --git a/location/tox.ini b/location/tox.ini index 3cd28ff4..504de885 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -8,6 +8,11 @@ wheel_build_env = .pkg deps = pytest>=6 pytest-cov + # The packages it depends on come from this checkout rather than a + # package index, so the tests run with the code they are released with. + -e../pipeline/fiftyone_pipeline_core/ + -e../pipeline/fiftyone_pipeline_engines/ + -e../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_location {tty:--color=yes} {posargs} pass_env = From e14dcda99ecaa220d4f1c0e37bfa8046ef50b20c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 00:23:38 +0000 Subject: [PATCH 15/26] Use a name in the object name test that a session id cannot contain (#44) --- .../tests/test_javascriptbuilder_object_name.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py index 92f95380..8342461f 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_object_name.py @@ -151,7 +151,9 @@ def test_valid_name_from_evidence_minified(self): @parameterized.expand([ ["statement", "a;b", True], - ["leading_digit", "9bad", True], + # The name has a letter that is not a hex digit, so the random + # session id the script also carries can never contain it. + ["leading_digit", "9zz", True], ["quote", 'x"y', True], ["empty", "", False], # A reserved word is also an ordinary word in the script's comments, From c7c45b5186d2b65ceafa9b5ab5b46eae14676c11 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 01:08:14 +0000 Subject: [PATCH 16/26] Run every example on the free data files, and show what needs the Enterprise data file (#43) --- .../readme.md | 19 ++ .../example_utils.py | 129 +++++++++--- .../onpremise/csv_profile_metadata.py | 15 +- .../onpremise/datafileupdate_console.py | 47 ++++- .../onpremise/gettingstarted_console.py | 35 +++- .../gettingstarted_web/templates/index.html | 26 +-- .../onpremise/match_metrics.py | 17 +- .../onpremise/metadata_console.py | 11 +- .../onpremise/offlineprocessing.py | 62 ++++-- .../onpremise/performance.py | 22 +- .../onpremise/profile_metadata_console.py | 15 +- .../templates/index.html | 18 +- .../tests/test_examples_without_settings.py | 140 +++++++++++++ ip-intelligence/ci/fetch-assets.ps1 | 5 + .../examples/gettingstarted.py | 2 +- .../readme.md | 36 ++-- .../example_utils.py | 192 +++++++++++++++--- .../onpremise/compare_console.py | 26 ++- .../onpremise/countries_console.py | 28 ++- .../onpremise/datafileupdate_console.py | 51 +++-- .../onpremise/gettingstarted_console.py | 53 +++-- .../onpremise/gettingstarted_web/app.py | 10 + .../onpremise/metadata_console.py | 24 ++- .../onpremise/mixed_console.py | 51 +++-- .../onpremise/offlineprocessing.py | 29 ++- .../onpremise/performance_console.py | 13 +- .../tests/test_examples_without_settings.py | 133 ++++++++++++ 27 files changed, 1002 insertions(+), 207 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index c668d1d5..d25b881e 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -16,12 +16,31 @@ getting started example: `python -m fiftyone_devicedetection_examples.cloud.gettingstarted_console` +The on-premise examples use the free Lite data file by default. The data +repository, at `cxx/data`, links to it, and an example downloads it the +first time it is missing, so the examples run from a fresh clone with no +settings. Where an example shows a property that only the Enterprise data +file has, such as HardwareVendor, HardwareName or DeviceType, it shows the +property as needing the Enterprise data file and gives a link to +[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-readme.md&utm_term=enterprise-data-file) +about a copy. Supply the path to an Enterprise data file as the first +command line argument to see every property. + +The data file update example needs a licence key, and the hardware profile +examples need the TAC data file or its CSV export, so without them each one +says what it needs and where to ask for it, and stops. + ## Tests To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +The tests in `tests/test_examples_without_settings.py` run the on-premise +examples as they run from a fresh clone, with no licence key, and check that +each one finishes on the free Lite data file and gives the link to ask for +the Enterprise data file. + Cloud tests will only run with a valid 51Degrees resource key (see above) set as a resource_key operating system environment variable. For example, use following command to set resource_key on Linux: diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py index 05260f95..2a50c478 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/example_utils.py @@ -24,6 +24,7 @@ from threading import Event, Thread import os import shlex +import shutil import subprocess from pathlib import Path @@ -59,9 +60,16 @@ class ExampleUtils: DATA_FOLDER = os.path.join("cxx", "data", "device-detection") # Sent with each download, so the downloads these examples make can be - # counted. + # counted. It is left out inside GitHub Actions, where the data + # repository's script names the build instead, so that test runs are + # not counted as people using the examples. DOWNLOAD_SOURCE = "examples-51degrees-python" + # Shown in place of a value for a property that the data file in use + # does not have. Every property the examples ask for is in the + # Enterprise data file. + NEEDS_ENTERPRISE = "needs the Enterprise data file" + @staticmethod def get_resource_key_from_config(config): key = "" @@ -246,14 +254,16 @@ def __find_file(file_name, stop_event, result, search_dir=None): if search_dir is None: search_dir = Path(os.getcwd()).absolute() - # Search for the file in the current directory - files = list(search_dir.rglob(_file.name)) - - # If the file is found in the current directory - # then set the result and return - if len(files): - result[0] = str(files[0].absolute()) - return + # Search the current directory and every folder below it. The stop + # event is checked in each folder, so the search ends soon after it + # times out, and a folder that cannot be read, such as one under + # /proc, is skipped. + for folder, _, names in os.walk(search_dir): + if stop_event.is_set(): + return + if _file.name in names: + result[0] = str(Path(folder, _file.name).absolute()) + return # If the file is not found in the current directory # then move to the parent directory and repeat the process @@ -271,23 +281,34 @@ def __find_file(file_name, stop_event, result, search_dir=None): def __download(path): name = os.path.basename(path) data = os.path.dirname(os.path.dirname(path)) + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + environment = None if os.name == "nt": + # PowerShell 7 when it is on the path, otherwise Windows + # PowerShell, which, started from PowerShell 7, inherits its + # module path and then cannot load its own commands. + power_shell = "pwsh" if shutil.which("pwsh") else "powershell" command = [ - "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", - "-File", os.path.join(data, "get-data-files.ps1"), name, - "-Source", ExampleUtils.DOWNLOAD_SOURCE] + power_shell, "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name] + if not in_actions: + command += ["-Source", ExampleUtils.DOWNLOAD_SOURCE] + if power_shell == "powershell": + environment = dict(os.environ) + environment.pop("PSModulePath", None) text = subprocess.list2cmdline(command) else: - command = [ - "sh", os.path.join(data, "get-data-files.sh"), - "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + command = ["sh", os.path.join(data, "get-data-files.sh")] + if not in_actions: + command += ["-s", ExampleUtils.DOWNLOAD_SOURCE] + command.append(name) text = shlex.join(command) # Flushed first, because the script writes to the same output. print(f"{name} is missing, so it is being downloaded by running\n" f"{text}", flush=True) try: - subprocess.run(command, check=False) + subprocess.run(command, check=False, env=environment) except OSError: pass if os.path.isfile(path): @@ -310,8 +331,70 @@ def data_file_is_old(engine): def get_data_file_tier(engine): return engine.engine.getProduct() + @staticmethod + def is_lite_data_file(engine): + """! + True when the engine uses the free Lite data file. + """ + return ExampleUtils.get_data_file_tier(engine) == "Lite" + + @staticmethod + def has_property(engine, name): + """! + True when the engine's data file has the property. The Lite data + file leaves out the properties only the Enterprise data file has, + such as HardwareVendor, HardwareName and DeviceType. The data + file's own list is read, because the engine's properties are only + those it was built to give. + """ + properties = engine.engine.getMetaData().getProperties() + name = name.lower() + for index in range(properties.getSize()): + if properties.getByIndex(index).getName().lower() == name: + return True + return False + + @staticmethod + def get_enterprise_value(engine, device, property): + """! + Format a property for display, or say that it needs the Enterprise + data file when the engine's data file does not have it. The value + is only read when the data file has the property. + """ + if not ExampleUtils.has_property(engine, property): + return ExampleUtils.NEEDS_ENTERPRISE + return ExampleUtils.get_human_readable(device, property) + + @staticmethod + def get_enterprise_note(contact_us): + """! + Where to ask for a copy of the Enterprise data file, written after an + example has shown properties that the data file in use does not + have. The link is tagged with the example's file, so that an enquiry + can be traced to the example. + """ + return ("Some of these properties need the Enterprise data file. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def get_enterprise_offer(contact_us): + """! + What the Enterprise data file adds and where to ask for a copy, + written after an example that shows no properties has run on the + free Lite data file. + """ + return ("This example used the free Lite data file. The Enterprise " + "data file adds many more properties, such as " + "HardwareVendor, HardwareName and DeviceType. Contact us for " + f"a copy at {contact_us}") + @staticmethod def check_data_file(pipeline, logger): + """! + Log the data file in use, and a warning if it is old. An example run + on the free Lite data file says what the Enterprise data file adds + in its own output. + """ # Get the 'engine' element within the pipeline that # performs device detection. We can use this to get # details about the data file as well as meta-data @@ -337,14 +420,6 @@ def check_data_file(pipeline, logger): "cxx/data/get-data-files.sh -u 51Degrees-LiteV4.1.hash, " + "or cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash " + "-Update on Windows. " + - "Find out about the Enterprise data file, which " + - "includes automatic updates, on our pricing page: " + - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") - - if ExampleUtils.get_data_file_tier(engine) == "Lite": - logger.log("warning", - "This example is using the 'Lite' " + - "data file. This is used for illustration, and " + - "has limited accuracy and capabilities. Find " + - "out about the Enterprise data file on our " + - "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=lite-data-file") + "Contact us about the Enterprise data file, which " + + "includes automatic updates, at " + + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-example_utils.py&utm_term=data-file-age-warning") diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py index da30fdbe..c4c78c15 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/csv_profile_metadata.py @@ -40,7 +40,6 @@ import sys from fiftyone_devicedetection_examples.example_utils import ExampleUtils -from fiftyone_pipeline_core.logger import Logger # CSV file name to search for CSV_FILE_NAME = "51Degrees-Tac-HardwarePlatform.csv" @@ -52,6 +51,10 @@ # Columns whose values are pipe-delimited lists MULTI_VALUE_COLUMNS = ["TAC", "HardwareName", "NativeName", "NativeModel"] +# Where to ask for a copy of the hardware profile export, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-csv_profile_metadata.py&utm_term=enterprise-data-file" + def flush_print(*args, **kwargs): print(*args, **kwargs, flush=True) @@ -110,15 +113,17 @@ def main(argv): output_file = None if output_arg == "-" else output_arg limit = int(argv[2]) if len(argv) > 2 else None - logger = Logger(min_level="info") output_func = flush_print if output_file else stderr_print if csv_file is not None: run(csv_file, output_file, output_func, limit) else: - logger.log("error", - "Failed to find the CSV data file. " - "Please provide the path to 51Degrees-Tac-HardwarePlatform.csv as the first argument.") + # The export is not one of the free data files, so the example says + # what it needs and where to ask for it, and stops. + print(f"Listing the hardware profiles needs the {CSV_FILE_NAME} " + "export, which is not one of the free data files. Give its " + "path as the first command line argument, or contact us for a " + f"copy at {CONTACT_US}") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py index 4fd126fc..c6d0e6a5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/datafileupdate_console.py @@ -32,9 +32,9 @@ # - Daily auto-update # # ## License Key -# In order to test this example you will need a 51Degrees Enterprise license which can be -# purchased from our [pricing page](https://51degrees.com/pricing/annual?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top). Look for our "Bigger" or -# "Biggest" options. +# Updating the data file needs a 51Degrees licence key for the Enterprise data file. +# Without one, the example shows the free Lite data file and says where to ask for a key. +# Contact us for a key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top # # # Data Files # You can find out more about data files, licenses etc. at our [FAQ page](https://51degrees.com/resources/faqs?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=top) @@ -166,10 +166,14 @@ from fiftyone_devicedetection_shared.key_utils import KeyUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +# Where to ask for a licence key and a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" DEFAULT_DATA_FILENAME = os.path.expanduser("~") + os.path.sep + ENTERPRISE_DATAFILE_NAME @@ -187,6 +191,32 @@ def clear(self): super().clear() class DataFileUpdateConsole(): + def explain_license_key(self, data_file, output): + # Without a licence key the example cannot download a data file, so + # it shows the data file it would start from, the free Lite file + # unless another is given, and says what an update needs. + if not data_file: + data_file = ExampleUtils.find_file(LITE_DATAFILE_NAME) + if data_file and os.path.isfile(data_file): + pipeline = DeviceDetectionPipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + with pipeline: + engine = pipeline.get_element("device") + output(f"The data file in use is the " + f"'{ExampleUtils.get_data_file_tier(engine)}' file published on " + f"{ExampleUtils.get_data_file_date(engine):%Y-%m-%d}, at " + f"'{data_file}'.") + output("") + output("Updating the data file downloads the Enterprise data file, " + "which needs a licence key. Give the key as the first command line " + f"argument, or in the environment variable " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key and a " + f"copy at {CONTACT_US}") + def run(self, data_file, license_key, interactive, logger, output): logger.log("info", "Starting example") @@ -195,13 +225,8 @@ def run(self, data_file, license_key, interactive, logger, output): license_key = KeyUtils.get_named_key(UPDATE_EXAMPLE_LICENSE_KEY_NAME) if not license_key: - logger.log("error", - "In order to test this example you will need a 51Degrees Enterprise " - "license which can be obtained on a trial basis or purchased from our\n" - "pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=license-key-required. You must supply the license " - "key as an argument to this program, or as an environment or system variable " - f"named '{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'") - raise Exception("No license key available") + self.explain_license_key(data_file, output) + return # work out where the downloaded file will be put, directory must exist if (data_file != None): @@ -283,7 +308,7 @@ def run(self, data_file, license_key, interactive, logger, output): # to notify when update complete data_file_update_service = update_service, # For automatic updates to work you will need to provide a license key. - # A license key can be obtained with a subscription from https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=run + # Contact us for a licence key at https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-datafileupdate_console.py&utm_term=run licence_keys = license_key, # Enable update on startup, the auto update system # will be used to check for an update before the diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 3675fee2..12c4cff6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -45,6 +45,17 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=enterprise-data-file" + +# The properties only the Enterprise data file has, with the names they are +# shown with. The free Lite data file has the ones the example shows first. +ENTERPRISE_PROPERTIES = [ + ("Hardware Vendor", "hardwarevendor"), + ("Hardware Name", "hardwarename"), + ("Device Type", "devicetype")] + class GettingStartedConsole(): def run(self, data_file, logger, output): @@ -76,8 +87,12 @@ def run(self, data_file, logger, output): ExampleUtils.check_data_file(pipeline, logger) # carry out some sample detections + needs_enterprise = False for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + needs_enterprise |= self.analyseEvidence( + values, pipeline, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) def analyseEvidence(self, evidence, pipeline, output): @@ -126,7 +141,18 @@ def analyseEvidence(self, evidence, pipeline, output): self.outputValue("Platform Version", device.platformversion, message) self.outputValue("Browser Name", device.browsername, message) self.outputValue("Browser Version", device.browserversion, message) + + # The engine says which properties its data file has, and the + # value of a property it does not have is not read. + engine = pipeline.get_element("device") + needs_enterprise = False + for name, property in ENTERPRISE_PROPERTIES: + value = ExampleUtils.get_enterprise_value( + engine, device, property) + needs_enterprise |= value == ExampleUtils.NEEDS_ENTERPRISE + message.append(f"\t{name}: {value}\n") output("".join(message)) + return needs_enterprise def outputValue(self, name, value, message): # Individual result values have a wrapper called @@ -147,9 +173,10 @@ def main(argv): # a command line parameter. # # Note that the Lite data file is only used for illustration, and has - # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main + # limited accuracy and capabilities. The example shows the properties + # only the Enterprise data file has as needing it. Contact us for a copy + # of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html index d26ab8da..a1352ee8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_web/templates/index.html @@ -32,8 +32,8 @@ days old. A more recent data file may be needed to correctly detect the latest devices, browsers, etc. Get the latest lite data file with the scripts in the data - repository on GitHub. Find out about the Enterprise data file, which includes automatic - daily updates, on our pricing page. + repository on GitHub. Contact us + about the Enterprise data file, which includes automatic daily updates. {% endif %}

    Web integration example

    @@ -130,17 +130,17 @@

    Device data

    - Hardware Vendor:{{ utils.get_human_readable(data.device, "hardwarevendor") }} - Hardware Name:{{ utils.get_human_readable(data.device, "hardwarename") }} - Device Type:{{ utils.get_human_readable(data.device, "devicetype") }} - Platform Vendor:{{ utils.get_human_readable(data.device, "platformvendor") }} - Platform Name:{{ utils.get_human_readable(data.device, "platformname") }} - Platform Version:{{ utils.get_human_readable(data.device, "platformversion") }} - Browser Vendor:{{ utils.get_human_readable(data.device, "browservendor") }} - Browser Name:{{ utils.get_human_readable(data.device, "browsername") }} - Browser Version:{{ utils.get_human_readable(data.device, "browserversion") }} - Screen width (pixels):{{ utils.get_human_readable(data.device, "screenpixelswidth") }} - Screen height (pixels):{{ utils.get_human_readable(data.device, "screenpixelsheight") }} + Hardware Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarevendor") }} + Hardware Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarename") }} + Device Type:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "devicetype") }} + Platform Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformvendor") }} + Platform Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformname") }} + Platform Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformversion") }} + Browser Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browservendor") }} + Browser Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browsername") }} + Browser Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browserversion") }} + Screen width (pixels):{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "screenpixelswidth") }} + Screen height (pixels):{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "screenpixelsheight") }} {# The device id identifies the matched hardware, platform and browser profiles, so it is the compact form of everything else in this table. #} Device Id:{{ utils.get_human_readable(data.device, "deviceid") }} diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index 6a223d00..dd53ed68 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -48,6 +48,10 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=enterprise-data-file" + class MatchMetricsConsole(): def run(self, data_file, show_descs, logger, output): @@ -76,8 +80,9 @@ def run(self, data_file, show_descs, logger, output): # Uncomment BrowserName to include Browser component profile ID # in the device ID value. #restricted_properties=["ismobile", "hardwarename", "browsername"], - # If using the full on-premise data file this property will be - # present in the data file. See https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run + # Only the Enterprise data file has this property, and the free + # Lite data file leaves it out. Contact us for a copy at + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run # Only use the predictive graph to better handle variances # between the training data and the target User-Agent string. # For a more detailed description of the differences between @@ -169,7 +174,9 @@ def get_component(property): if (show_descs == True): output(f" {propertyDescription}") - output() + output("") + if ExampleUtils.is_lite_data_file(pipeline.get_element("device")): + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) logger.log("info", "Finished Match Metrics Example") # Evidence values from a windows 11 device using a browser @@ -183,8 +190,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py index 5e605158..e3141e06 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py @@ -50,6 +50,10 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_VALUES from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=enterprise-data-file" + bgRed = "\u001b[41m" fgWhite = "\u001b[37;1m" colReset = "\u001b[0m" @@ -84,6 +88,9 @@ def run(self, data_file, logger, output): self.outputEvidenceKeyDetails(pipeline.get_element("device"), output) ExampleUtils.check_data_file(pipeline, logger) + if ExampleUtils.is_lite_data_file(pipeline.get_element("device")): + output("") + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) def outputEvidenceKeyDetails(self, engine, output): @@ -121,8 +128,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py index 500fdba7..c4b437f5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py @@ -59,6 +59,22 @@ from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME from ruamel.yaml import YAML +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=enterprise-data-file" + +# The properties only the Enterprise data file has, which are recorded as +# needing it when the data file in use does not have them. +ENTERPRISE_PROPERTIES = [ + "tac", + "devicetype", + "hardwarevendor", + "hardwarename", + "hardwaremodel", + "nativedevice", + "nativename", + "nativemodel"] + class OfflineProcessing(): def run(self, data_file, evidence_yaml, logger, output): """! @@ -68,6 +84,8 @@ def run(self, data_file, evidence_yaml, logger, output): @param evidence_yaml: File containing the yaml representation of the evidence to process @param logger: Logger to use within the pipeline @param output: Output file to write results to + @return: True when the data file lacks properties that only the + Enterprise data file has """ # In this example, we use the DeviceDetectionPipelineBuilder @@ -108,7 +126,12 @@ def run(self, data_file, evidence_yaml, logger, output): # and some evidence files contain them in user-agent strings. content = evidence_yaml.read().replace('\t', ' ') yaml_data = yaml.load_all(content) - + # The properties only the Enterprise data file has that the data + # file in use lacks, found once rather than for every record. + engine = pipeline.get_element("device") + missing = [name for name in ENTERPRISE_PROPERTIES + if not ExampleUtils.has_property(engine, name)] + try: # Keep going as long as we have more document records. for evidence in yaml_data: @@ -126,18 +149,25 @@ def run(self, data_file, evidence_yaml, logger, output): # Keys already have header. prefix in the YAML file headers[key] = str(evidence[key]) - self.analyseEvidence(headers, pipeline, output, yaml) + self.analyseEvidence( + headers, pipeline, output, yaml, missing) except BaseException as err: # We can't read the evidence values, so cant write them to the output. Will just # have to skip this entry. logger.log("error", err) + if missing: + # A comment keeps the output a valid YAML stream. + print("# " + ExampleUtils.get_enterprise_note(CONTACT_US), + file = output) + # write the yaml document end marker print("...", file = output) ExampleUtils.check_data_file(pipeline, logger) + return len(missing) > 0 - def analyseEvidence(self, evidence, pipeline, output, yaml): + def analyseEvidence(self, evidence, pipeline, output, yaml, missing): # FlowData is a data structure that is used to convey information required for # detection and the results of the detection through the pipeline. # Information required for detection is called "evidence" and usually consists @@ -165,14 +195,12 @@ def analyseEvidence(self, evidence, pipeline, output, yaml): values["device.platformversion"] = ExampleUtils.get_human_readable(device, "platformversion") values["device.browsername"] = ExampleUtils.get_human_readable(device, "browsername") values["device.browserversion"] = ExampleUtils.get_human_readable(device, "browserversion") - values["device.tac"] = ExampleUtils.get_human_readable(device, "tac") - values["device.devicetype"] = ExampleUtils.get_human_readable(device, "devicetype") - values["device.hardwarevendor"] = ExampleUtils.get_human_readable(device, "hardwarevendor") - values["device.hardwarename"] = ExampleUtils.get_human_readable(device, "hardwarename") - values["device.hardwaremodel"] = ExampleUtils.get_human_readable(device, "hardwaremodel") - values["device.nativedevice"] = ExampleUtils.get_human_readable(device, "nativedevice") - values["device.nativename"] = ExampleUtils.get_human_readable(device, "nativename") - values["device.nativemodel"] = ExampleUtils.get_human_readable(device, "nativemodel") + # The value of a property the data file does not have is not + # read. + for property in ENTERPRISE_PROPERTIES: + values["device." + property] = ( + ExampleUtils.NEEDS_ENTERPRISE if property in missing + else ExampleUtils.get_human_readable(device, property)) # DeviceId is a unique identifier for the combination of hardware, operating # system, browser and crawler that has been detected. # Our device detection solution uses machine learning to find the optimal @@ -196,8 +224,8 @@ def main(argv): # # Note that the Lite data file is only used for illustration, and has # limited accuracy and capabilities. - # Find out about the Enterprise data file on our pricing page: - # https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main + # Contact us for a copy of the Enterprise data file: + # https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # This file contains the 20,000 most commonly seen combinations of header values # that are relevant to device detection. For example, User-Agent and UA-CH headers. @@ -212,9 +240,11 @@ def main(argv): if (data_file != None): with open(output_file, "w") as output: with open(evidence_file, "r") as input: - OfflineProcessing().run(data_file, input, logger, output) - logger.log("info", - f"Processing complete. See results in: '{output_file}'") + needs_enterprise = OfflineProcessing().run( + data_file, input, logger, output) + print(f"Processing complete. See results in: '{output_file}'") + if needs_enterprise: + print(ExampleUtils.get_enterprise_note(CONTACT_US)) else: logger.log("error", "Failed to find a device detection data file. Make sure the " + diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py index d795b548..000186b4 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py @@ -51,6 +51,11 @@ from fiftyone_devicedetection_examples.example_utils import ExampleUtils # This example goes through a CSV of 20000 user agents and processes them, returning the time and information about the matches from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder +from fiftyone_devicedetection_shared.example_constants import UA_FILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-performance.py&utm_term=enterprise-data-file" # Here we make a function that processes a user agent # And returns if it is a mobile device @@ -150,11 +155,13 @@ def run(data_file, skip = False): if __name__ == "__main__": ap = argparse.ArgumentParser(description='Run detection benchmark.') ap.add_argument('-d', '--data_file', default='', help='Path to data file') - ap.add_argument('-u', '--user_agents_file', default='../../cxx/data/device-detection/20000 User Agents.csv', help='Path to user agents evidence file') + ap.add_argument('-u', '--user_agents_file', default='', help='Path to user agents evidence file') ap.add_argument('-j', '--json_output', default='', help='Output results in JSON format') args = ap.parse_args() if args.data_file == "": args.data_file = ExampleUtils.find_file("51Degrees-LiteV4.1.hash") + if args.user_agents_file == "": + args.user_agents_file = ExampleUtils.find_file(UA_FILE_NAME) # First we read the contents of the 20000 user agents file as a list with open(args.user_agents_file, newline='') as file: @@ -206,3 +213,16 @@ def run(data_file, skip = False): final_result["notmobile"] += result["notmobile"] print("Results", final_result) + + # Say what the Enterprise data file adds when the benchmark ran on the + # free Lite data file. The data file is opened with the low memory + # profile, which reads little of it. + with DeviceDetectionOnPremisePipelineBuilder( + data_file_path=args.data_file, + licence_keys="", + performance_profile='LowMemory', + add_javascript_builder=False, + usage_sharing=False, + auto_update=False).build() as check: + if ExampleUtils.is_lite_data_file(check.get_element("device")): + print(ExampleUtils.get_enterprise_offer(CONTACT_US)) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py index 60fe8f6d..015c35d3 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/profile_metadata_console.py @@ -61,6 +61,11 @@ def stderr_print(*args, **kwargs): print(*args, **kwargs, file=sys.stderr, flush=True) +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-profile_metadata_console.py&utm_term=enterprise-data-file" + + class ProfileMetadataConsole(): # Properties we want to extract from each hardware profile # Note: TAC, HardwareName, NativeName, NativeModel are multi-value properties (string[]) @@ -199,9 +204,13 @@ def main(argv): if data_file is not None: ProfileMetadataConsole().run(data_file, output_file, logger, output_func, limit) else: - logger.log("error", - "Failed to find a device detection data file. " + - "Please provide the path to a TAC-HashV41.hash file as the first argument.") + # Only the Enterprise data file has the properties the example lists, + # so without one it says so and where to ask for one, and stops. + print("Listing the hardware profiles needs the TAC, HardwareVendor " + "and HardwareName properties, which only the Enterprise data " + "file has. Give the path to an Enterprise data file as the " + "first command line argument, or contact us for a copy at " + f"{CONTACT_US}") if __name__ == "__main__": diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html index a8c19cd5..31a88034 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/useragentclienthints_web/templates/index.html @@ -85,15 +85,15 @@

    Detection results

    - Hardware Vendor:{{ utils.get_human_readable(data.device, "hardwarevendor") }} - Hardware Name:{{ utils.get_human_readable(data.device, "hardwarename") }} - Device Type:{{ utils.get_human_readable(data.device, "devicetype") }} - Platform Vendor:{{ utils.get_human_readable(data.device, "platformvendor") }} - Platform Name:{{ utils.get_human_readable(data.device, "platformname") }} - Platform Version:{{ utils.get_human_readable(data.device, "platformversion") }} - Browser Vendor:{{ utils.get_human_readable(data.device, "browservendor") }} - Browser Name:{{ utils.get_human_readable(data.device, "browsername") }} - Browser Version:{{ utils.get_human_readable(data.device, "browserversion") }} + Hardware Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarevendor") }} + Hardware Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "hardwarename") }} + Device Type:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "devicetype") }} + Platform Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformvendor") }} + Platform Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformname") }} + Platform Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "platformversion") }} + Browser Vendor:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browservendor") }} + Browser Name:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browsername") }} + Browser Version:{{ utils.get_enterprise_value(data.pipeline.get_element("device"), data.device, "browserversion") }} diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py new file mode 100644 index 00000000..c035294d --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_examples_without_settings.py @@ -0,0 +1,140 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Runs the on-premise examples as they run from a fresh clone, with no +# arguments and no licence key, so that they use the free Lite data file. +# Each must finish, show the properties only the Enterprise data file has +# as needing it, and give the link to ask for a copy. + +import io +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_pipeline_core.logger import Logger + +from fiftyone_devicedetection_examples.example_utils import ExampleUtils +from fiftyone_devicedetection_examples.onpremise import csv_profile_metadata +from fiftyone_devicedetection_examples.onpremise import datafileupdate_console +from fiftyone_devicedetection_examples.onpremise import gettingstarted_console +from fiftyone_devicedetection_examples.onpremise import match_metrics +from fiftyone_devicedetection_examples.onpremise import metadata_console +from fiftyone_devicedetection_examples.onpremise import offlineprocessing +from fiftyone_devicedetection_examples.onpremise import profile_metadata_console +from fiftyone_devicedetection_shared.example_constants import EVIDENCE_FILE_NAME +from fiftyone_devicedetection_shared.example_constants import LITE_DATAFILE_NAME + +# The settings the examples read, which a fresh clone does not have. +SETTINGS = ["license_key", "LICENSE_KEY"] + +# The evidence records the offline processing test reads. The example's own +# file has 20,000, which take minutes. +OFFLINE_RECORDS = 20 + + +def run_without_settings(main, argv=None): + """Runs an example's main with the settings taken out of the + environment, and returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + main(argv or []) + return printed.getvalue() + + +def first_records(path, count): + """The first records of a YAML evidence file.""" + lines = [] + with open(path, encoding="utf-8") as evidence: + for line in evidence: + if line.rstrip() == "---": + count -= 1 + if count < 0: + break + lines.append(line) + return io.StringIO("".join(lines)) + + +class ExamplesWithoutSettingsTests(unittest.TestCase): + + def test_gettingstarted_console(self): + printed = run_without_settings(gettingstarted_console.main) + self.assertIn( + "Hardware Vendor: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), + printed) + + def test_match_metrics(self): + printed = run_without_settings(match_metrics.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(match_metrics.CONTACT_US), + printed) + + def test_metadata_console(self): + printed = run_without_settings(metadata_console.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), + printed) + + def test_offline_processing(self): + output = io.StringIO() + lacks_properties = offlineprocessing.OfflineProcessing().run( + ExampleUtils.find_file(LITE_DATAFILE_NAME), + first_records( + ExampleUtils.find_file(EVIDENCE_FILE_NAME), OFFLINE_RECORDS), + Logger(min_level="error"), + output) + self.assertTrue(lacks_properties) + self.assertIn( + "device.hardwarevendor: " + ExampleUtils.NEEDS_ENTERPRISE, + output.getvalue()) + self.assertIn( + "# " + ExampleUtils.get_enterprise_note( + offlineprocessing.CONTACT_US), + output.getvalue()) + + def test_datafileupdate_console(self): + printed = run_without_settings(datafileupdate_console.main) + self.assertIn("The data file in use is the 'Lite' file", printed) + self.assertIn(datafileupdate_console.CONTACT_US, printed) + + # The two examples below need files that are not free, so in a fresh + # clone the search for them finds nothing, which the tests stand in for. + + def test_csv_profile_metadata(self): + with mock.patch.object(ExampleUtils, "find_file", return_value=None): + printed = run_without_settings(csv_profile_metadata.main) + self.assertIn(csv_profile_metadata.CSV_FILE_NAME, printed) + self.assertIn(csv_profile_metadata.CONTACT_US, printed) + + def test_profile_metadata_console(self): + with mock.patch.object(ExampleUtils, "find_file", return_value=None): + printed = run_without_settings(profile_metadata_console.main) + self.assertIn("only the Enterprise data file has", printed) + self.assertIn(profile_metadata_console.CONTACT_US, printed) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/ci/fetch-assets.ps1 b/ip-intelligence/ci/fetch-assets.ps1 index a26332bc..f4653aa8 100644 --- a/ip-intelligence/ci/fetch-assets.ps1 +++ b/ip-intelligence/ci/fetch-assets.ps1 @@ -31,6 +31,11 @@ foreach ($asset in $assets) { # The ASN file is not an asset, so the data repository's script downloads it # beside its link in cxx/data/ip-intelligence, where the tests look for it. +# The examples look for the Lite data file in the data repository, under the +# name its script saves it as, so the asset is linked there too rather than +# downloaded again. +New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/51Degrees-LiteIpiV41.ipi" -Path "$PSScriptRoot/../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" + $global:LASTEXITCODE = 0 & "$PSScriptRoot/../../cxx/data/get-data-files.ps1" 51Degrees-IPIV4AsnIpiV41.ipi if ($LASTEXITCODE -ne 0) { diff --git a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py index a2511324..0450b9eb 100644 --- a/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py +++ b/ip-intelligence/fiftyone_ipintelligence/examples/gettingstarted.py @@ -24,7 +24,7 @@ # First create the IP intelligence pipeline with the desired settings. -data_file = "../assets/51Degrees-LiteIpiV41.ipi" +data_file = "../../cxx/data/ip-intelligence/51Degrees-LiteV41.ipi" resource_key = "!!YOUR_RESOURCE_KEY!!" # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md index c41c0630..dd0f3423 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_examples/readme.md @@ -33,23 +33,32 @@ python -m fiftyone_ipintelligence_examples.cloud.gettingstarted_web ## On-premise examples -Require a `.ipi` data file in the format the engine reads (see +Use a `.ipi` data file in the format the engine reads (see [Data files](../readme.md#data-files)). The data repository, at -`cxx/data`, links to a freely available network-only file -(`51Degrees-IPIV4AsnIpiV41.ipi`) that the examples fall back to, and an -example downloads it when it is missing. It only contains the ASN/network -properties, so location and registration properties are reported as "not -present in the data file". - -For the full set of properties (location, registration, etc.), a more -complete data file is required - to obtain one for testing please -[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=on-premise-examples) - and supply its path as -the first command line argument. +`cxx/data`, links to the two free data files, and an example downloads the +one it uses the first time it is missing, so the examples run from a fresh +clone with no settings. The free Lite data file (`51Degrees-LiteV41.ipi`) +has the registration properties, such as RegisteredName and +RegisteredCountry, and most examples use it. The free ASN data file +(`51Degrees-IPIV4AsnIpiV41.ipi`) has the ASN properties and is far smaller, +so the performance and data file update examples use it. + +The location, range and connection properties are only in the Enterprise +data file. An example run on a free data file shows each property it lacks +as needing the Enterprise data file, and gives a link to +[contact us](https://51degrees.com/contact-us?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-readme.md&utm_term=on-premise-examples) about a copy. +The countries and comparison examples need those properties throughout, so +on a free data file they say which they need and stop. Supply the path to +an Enterprise data file as the first command line argument, or with `-d`, +to see every property. - `onpremise/gettingstarted_console.py` - resolve IP addresses from the console using a local data file. - `onpremise/metadata_console.py` - list the properties available from the data file. +- `onpremise/countries_console.py` - list the countries an IP address is + most likely in, with their names translated (needs the Enterprise data + file). - `onpremise/offlineprocessing.py` - batch process the evidence file from the data repository. - `onpremise/performance_console.py` - benchmark the number of IP lookups @@ -57,7 +66,7 @@ the first command line argument. - `onpremise/compare_console.py` - compare the location returned for each IP address against a CSV of known-true locations. - `onpremise/datafileupdate_console.py` - demonstrate automatic data file - updates (requires a license key). + updates (needs a licence key, and without one says where to ask for one). - `onpremise/mixed_console.py` - combine device detection and IP intelligence in a single on-premise pipeline (requires the device detection package and a `.hash` data file). @@ -67,10 +76,11 @@ the first command line argument. ``` python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console [data-file] python -m fiftyone_ipintelligence_examples.onpremise.metadata_console [data-file] +python -m fiftyone_ipintelligence_examples.onpremise.countries_console [data-file] python -m fiftyone_ipintelligence_examples.onpremise.offlineprocessing [data-file [evidence-file]] python -m fiftyone_ipintelligence_examples.onpremise.performance_console [-d data-file] [-e evidence-file] python -m fiftyone_ipintelligence_examples.onpremise.compare_console [-d data-file] [-t truth-file] -python -m fiftyone_ipintelligence_examples.onpremise.datafileupdate_console [data-file] +python -m fiftyone_ipintelligence_examples.onpremise.datafileupdate_console [license-key [data-file]] python -m fiftyone_ipintelligence_examples.onpremise.mixed_console [hash-file [ipi-file]] python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_web ``` diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py index 81eb5829..3ffd5499 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py @@ -23,6 +23,7 @@ from datetime import datetime, timedelta import os import shlex +import shutil import subprocess from pathlib import Path @@ -50,10 +51,21 @@ class ExampleUtils: # belongs in this folder, beside its .link file. DATA_FOLDER = os.path.join("cxx", "data", "ip-intelligence") + # The data repository's device detection folder, for the examples that + # use both products. + DEVICE_DETECTION_DATA_FOLDER = os.path.join("cxx", "data", "device-detection") + # Sent with each download, so the downloads these examples make can be - # counted. + # counted. It is left out inside GitHub Actions, where the data + # repository's script names the build instead, so that test runs are + # not counted as people using the examples. DOWNLOAD_SOURCE = "examples-51degrees-python" + # Shown in place of a value for a property that the data file in use + # does not have. Every property the examples ask for is in the + # Enterprise data file. + NEEDS_ENTERPRISE = "needs the Enterprise data file" + @staticmethod def get_human_readable(ip_data, property): try: @@ -124,7 +136,7 @@ def find_file(file_name): Find the specified file. First checks the data file environment variable, then searches up the directory tree from the current working directory looking in 'assets' and the data repository's - folder. A file the data repository links to is downloaded there + folders. A file the data repository links to is downloaded there when it is missing. """ @@ -134,17 +146,20 @@ def find_file(file_name): current = Path(os.getcwd()).resolve() roots = [current, *current.parents] + folders = [ + ExampleUtils.DATA_FOLDER, + ExampleUtils.DEVICE_DETECTION_DATA_FOLDER] for root in roots[:8]: - for sub_dir in [".", "assets", ExampleUtils.DATA_FOLDER]: + for sub_dir in [".", "assets", *folders]: candidate = os.path.join(str(root), sub_dir, file_name) if os.path.isfile(candidate): return candidate for root in roots[:8]: - candidate = os.path.join( - str(root), ExampleUtils.DATA_FOLDER, file_name) - if os.path.isfile(candidate + ".link"): - return ExampleUtils.__download(candidate) + for folder in folders: + candidate = os.path.join(str(root), folder, file_name) + if os.path.isfile(candidate + ".link"): + return ExampleUtils.__download(candidate) return None @@ -160,23 +175,34 @@ def __download(path): name = os.path.basename(path) data = os.path.dirname(os.path.dirname(path)) + in_actions = os.environ.get("GITHUB_ACTIONS") == "true" + environment = None if os.name == "nt": + # PowerShell 7 when it is on the path, otherwise Windows + # PowerShell, which, started from PowerShell 7, inherits its + # module path and then cannot load its own commands. + power_shell = "pwsh" if shutil.which("pwsh") else "powershell" command = [ - "powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", - "-File", os.path.join(data, "get-data-files.ps1"), name, - "-Source", ExampleUtils.DOWNLOAD_SOURCE] + power_shell, "-NoProfile", "-ExecutionPolicy", "Bypass", + "-File", os.path.join(data, "get-data-files.ps1"), name] + if not in_actions: + command += ["-Source", ExampleUtils.DOWNLOAD_SOURCE] + if power_shell == "powershell": + environment = dict(os.environ) + environment.pop("PSModulePath", None) text = subprocess.list2cmdline(command) else: - command = [ - "sh", os.path.join(data, "get-data-files.sh"), - "-s", ExampleUtils.DOWNLOAD_SOURCE, name] + command = ["sh", os.path.join(data, "get-data-files.sh")] + if not in_actions: + command += ["-s", ExampleUtils.DOWNLOAD_SOURCE] + command.append(name) text = shlex.join(command) # Flushed first, because the script writes to the same output. print(f"{name} is missing, so it is being downloaded by running\n" f"{text}", flush=True) try: - subprocess.run(command, check=False) + subprocess.run(command, check=False, env=environment) except OSError: pass if os.path.isfile(path): @@ -185,6 +211,132 @@ def __download(path): f"again.\n{text}") return None + @staticmethod + def has_property(engine, name): + + """! + True when the engine's data file has the property. The free Lite data + file has the registration properties, such as RegisteredName, and the + free ASN data file the ASN properties, such as AsnName. The data + file's own list is read, because the engine's properties are only + those it was built to give. + """ + + properties = engine.engine.getMetaData().getProperties() + name = name.lower() + for index in range(properties.getSize()): + if properties.getByIndex(index).getName().lower() == name: + return True + return False + + @staticmethod + def is_free_data_file(engine): + + """! + True when the engine uses one of the free data files, Lite or ASN. + Only the Enterprise data file has the range properties, so a check + for IpRangeStart tells them apart. + """ + + return not ExampleUtils.has_property(engine, "IpRangeStart") + + @staticmethod + def is_free_data_file_at(data_file): + + """! + True when the data file at the path is one of the free data files. + The data file is opened with the low memory profile, which reads + little of it. + """ + + with ExampleUtils.__open(data_file) as pipeline: + return ExampleUtils.is_free_data_file( + pipeline.flow_elements_list["ip"]) + + @staticmethod + def get_enterprise_value(engine, ip_data, property): + + """! + Format a property for display, or say that it needs the Enterprise + data file when the engine's data file does not have it. The value + is only read when the data file has the property. + """ + + if not ExampleUtils.has_property(engine, property): + return ExampleUtils.NEEDS_ENTERPRISE + return ExampleUtils.get_human_readable(ip_data, property) + + @staticmethod + def lacks_enterprise_properties(data_file, purpose, properties, output, + contact_us): + + """! + For an example that needs properties only the Enterprise data file + has, report when the data file has none of them, and then say so and + where to ask for a copy of the Enterprise data file. The data file is + opened with the low memory profile, which reads little of it. + @param purpose: What the example does, as the start of a sentence, + for example "Comparing known locations with IP Intelligence". + @param contact_us: The link to the contact page, tagged with the + example's file, so that an enquiry can be traced to the example. + """ + + with ExampleUtils.__open(data_file) as pipeline: + engine = pipeline.flow_elements_list["ip"] + if any(ExampleUtils.has_property(engine, name) + for name in properties): + return False + if len(properties) == 1: + names = properties[0] + else: + names = ", ".join(properties[:-1]) + " and " + properties[-1] + output(f"{purpose} needs the {names} properties, which only the " + "Enterprise data file has. The data file in use has none of " + "them, so the example stops here. Contact us for a copy of " + f"the Enterprise data file at {contact_us}") + return True + + @staticmethod + def get_enterprise_note(contact_us): + + """! + Where to ask for a copy of the Enterprise data file, written after an + example has shown properties that the data file in use does not + have. The link is tagged with the example's file, so that an enquiry + can be traced to the example. + """ + + return ("Some of these properties need the Enterprise data file. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def get_enterprise_offer(contact_us): + + """! + What the Enterprise data file adds and where to ask for a copy, + written after an example that shows no location properties has run + on a free data file. + """ + + return ("This example used a free data file. The Enterprise data " + "file adds the location, range and connection properties. " + f"Contact us for a copy at {contact_us}") + + @staticmethod + def __open(data_file): + + """! + A pipeline that reads the data file with the low memory profile. + """ + + from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder + return IpIntelligencePipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + @staticmethod def get_data_file_date(pipeline): engine = pipeline.flow_elements_list["ip"] @@ -194,8 +346,9 @@ def get_data_file_date(pipeline): def check_data_file(pipeline, logger): """! - Display information about the data file and log warnings if it - is old or a Lite file. + Display information about the data file and log a warning if it is + old. An example run on a free data file says what the Enterprise + data file adds in its own output. """ engine = pipeline.flow_elements_list["ip"] @@ -223,13 +376,6 @@ def check_data_file(pipeline, logger): f"or cxx/data/get-data-files.ps1 ip-intelligence -Update " f"on Windows.") - if product == "Lite": - logger.log("warning", - "This example is using the 'Lite' data file. This is " - "used for illustration, and has limited properties. A " - "more complete data file can be obtained from " - "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-example_utils.py&utm_term=lite-data-file") - @staticmethod def __get_env_variable(name): value = os.environ.get(name) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py index e418f786..d928f86f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/compare_console.py @@ -39,9 +39,10 @@ # true location and the returned location is then calculated, along with the # location confidence and whether an area was returned. # -# This example requires an enterprise IP intelligence data file (.ipi). To -# obtain an enterprise data file for testing, please contact us: -# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=top +# This example needs the location properties only the Enterprise IP +# intelligence data file has. With the free Lite data file, which it uses by +# default, it says which and stops. To obtain an Enterprise data file, please +# contact us: https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=top # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -54,11 +55,19 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME # Name of the truth CSV file in the data repository. GEOIP_COMPARISON_EVIDENCE_FILE_NAME = "geoip_comparison_evidence.csv" +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-compare_console.py&utm_term=enterprise-data-file" + +# The properties the comparison reads, which only the Enterprise data file +# has. +LOCATION_PROPERTIES = ["Latitude", "Longitude", "LocationConfidence", "Country"] + # Mean radius of the earth in kilometres, used for the distance calculation. EARTH_RADIUS_KM = 6371.0 @@ -175,12 +184,19 @@ def main(argv): help="Maximum number of truth records to process") args = ap.parse_args(argv) - data_file = args.data_file or ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + data_file = args.data_file or ExampleUtils.find_file(LITE_DATAFILE_NAME) truth_file = args.truth_file or ExampleUtils.find_file( GEOIP_COMPARISON_EVIDENCE_FILE_NAME) logger = Logger(min_level="info") + if data_file and ExampleUtils.lacks_enterprise_properties( + data_file, + "Comparing known locations with IP Intelligence", + LOCATION_PROPERTIES, + print, + CONTACT_US): + return if data_file and truth_file: with open(args.output_file, "w", newline="", encoding="utf-8") as fout: CompareConsole().run( diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py index eb239813..6e33ef2b 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/countries_console.py @@ -32,10 +32,10 @@ # 2. How to supply the browser language so the names are localized # 3. How to read the ordered "All" lists used to build a country dropdown # -# This example uses a local IP intelligence data file (.ipi). A data file that -# contains country information is needed to see weighted countries lead the -# list; with the network-only ASN data file the lists are simply every known -# country in alphabetical order. +# The country codes this example orders and translates are only in the +# Enterprise data file. With the free Lite data file, which it uses by +# default, the example says so and stops. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-countries_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -49,7 +49,15 @@ add_country_translation) from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_VALUES -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-countries_console.py&utm_term=enterprise-data-file" + +# The properties the country lists are built from, which only the Enterprise +# data file has. +COUNTRY_PROPERTIES = ["CountryCodesGeographical", "CountryCodesPopulation"] # The browser language to translate the country names to. A value such as # "fr_FR" or a full Accept-Language header is accepted; leave blank to keep the @@ -146,12 +154,18 @@ def main(argv): data_file = ( argv[0] if len(argv) > 0 - else ExampleUtils.find_file(ASN_DATAFILE_NAME)) + else ExampleUtils.find_file(LITE_DATAFILE_NAME)) logger = Logger(min_level="info") if data_file: - CountriesConsole().run(data_file, logger, print) + if not ExampleUtils.lacks_enterprise_properties( + data_file, + "Listing the countries an IP address is most likely in", + COUNTRY_PROPERTIES, + print, + CONTACT_US): + CountriesConsole().run(data_file, logger, print) else: logger.log("error", "Failed to find an IP intelligence data file. Make sure the " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index 37200afd..a11c17a3 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -32,10 +32,11 @@ # - Daily auto-update # # ## License Key -# In order to run this example you will need a 51Degrees Enterprise license, -# which can be purchased from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=top. -# Supply the license key as the first command line argument or in the -# 'license_key' environment variable. +# Updating the data file needs a 51Degrees licence key for the Enterprise +# data file. Without one, the example shows the free ASN data file and says +# where to ask for a key. Supply the licence key as the first command line +# argument or in the 'license_key' environment variable. Contact us for a key: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=header # # ## Update on start-up # Configure the pipeline to download an Enterprise data file on start-up. @@ -69,6 +70,7 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_pipeline_engines.datafile_update_service import DataFileUpdateService from fiftyone_pipeline_engines.datafile_update_service import UpdateStatus @@ -76,6 +78,10 @@ # a command line argument. UPDATE_EXAMPLE_LICENSE_KEY_NAME = "license_key" +# Where to ask for a licence key and a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=licence-key" + # The product type passed to the 51Degrees distributor. This identifies the # IP intelligence Enterprise product and may need to be adjusted to match # the product your license entitles you to. @@ -100,6 +106,33 @@ def clear(self): class DataFileUpdateConsole(): + def explain_license_key(self, data_file, output): + # Without a licence key the example cannot download a data file, so + # it shows the data file it would start from, the free ASN file + # unless another is given, and says what an update needs. + if not data_file: + data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) + if data_file and os.path.isfile(data_file): + pipeline = IpIntelligencePipelineBuilder( + data_file_path = data_file, + performance_profile = "LowMemory", + usage_sharing = False, + auto_update = False, + licence_keys = "").build() + with pipeline: + engine = pipeline.flow_elements_list["ip"].engine + published = engine.getPublishedTime() + date = datetime(published.getYear(), published.getMonth(), + published.getDay()) + output(f"The data file in use is the '{engine.getProduct()}' " + f"file published on {date:%Y-%m-%d}, at '{data_file}'.") + output("") + output("Updating the data file downloads the Enterprise data file, " + "which needs a licence key. Give the key as the first command " + "line argument, or in the environment variable " + f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'. Contact us for a key " + f"and a copy at {CONTACT_US}") + def run(self, data_file, license_key, interactive, logger, output): logger.log("info", "Starting example") @@ -108,14 +141,8 @@ def run(self, data_file, license_key, interactive, logger, output): license_key = os.environ.get(UPDATE_EXAMPLE_LICENSE_KEY_NAME) if not license_key: - logger.log("error", - "In order to run this example you will need a 51Degrees " - "Enterprise license, which can be obtained on a trial basis " - "or purchased from our pricing page https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-datafileupdate_console.py&utm_term=run. " - "You must supply the license key as an argument to this " - "program, or in an environment variable named " - f"'{UPDATE_EXAMPLE_LICENSE_KEY_NAME}'") - raise Exception("No license key available") + self.explain_license_key(data_file, output) + return # Work out where the downloaded file will be put. The directory must # exist; the file itself need not. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py index 4988da7d..70c2e121 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_console.py @@ -32,10 +32,12 @@ # 2. How to pass input data (evidence) to the Pipeline # 3. How to retrieve the results # -# This example uses a local IP intelligence data file (.ipi). The data -# repository links to a freely available data file, which the example -# downloads when it is missing. A more complete data file can be supplied -# as the first command line argument. +# By default the example uses the free Lite IP intelligence data file, which +# it downloads when it is missing. The Lite data file has the registration +# properties, and the example shows each property only the Enterprise data +# file has as needing it. An Enterprise data file can be supplied as the first +# command line argument. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -45,12 +47,15 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_VALUES -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=enterprise-data-file" # The properties displayed for each IP address. These mirror the .NET -# getting started example. Properties that are not present in the data file -# being used (for example the network-only ASN data file) are reported as -# such rather than causing an error. +# getting started example. A property the data file in use does not have is +# shown as needing the Enterprise data file. OUTPUT_PROPERTIES = [ "RegisteredName", "RegisteredOwner", @@ -99,8 +104,12 @@ def run(self, data_file, logger, output): ExampleUtils.check_data_file(pipeline, logger) # carry out some sample lookups + needs_enterprise = False for values in EVIDENCE_VALUES: - self.analyseEvidence(values, pipeline, output) + needs_enterprise |= self.analyseEvidence( + values, pipeline, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) def analyseEvidence(self, evidence, pipeline, output): @@ -141,11 +150,16 @@ def analyseEvidence(self, evidence, pipeline, output): # Display the results of the lookup. See the property # dictionary at https://51degrees.com/developers/property-dictionary?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_console.py&utm_term=analyseevidence # for details of all available properties. + # The engine says which properties its data file has. + engine = pipeline.flow_elements_list["ip"] + needs_enterprise = False for name in OUTPUT_PROPERTIES: - self.outputValue(ip_data, name, message) + needs_enterprise |= self.outputValue( + engine, ip_data, name, message) output("".join(message)) + return needs_enterprise - def outputValue(self, ip_data, name, message): + def outputValue(self, engine, ip_data, name, message): # Individual result values have a wrapper called # `AspectPropertyValue`. This functions similarly to # a null-able type. @@ -153,22 +167,23 @@ def outputValue(self, ip_data, name, message): # `value` method will throw an exception. # `AspectPropertyValue` also includes the `no_value_message` # method, which describes why the value has not been set. - # `get_value` returns None when the property is not present in the - # data file at all (for example the network-only ASN data file), - # rather than raising. - value = ExampleUtils.get_value(ip_data, name.lower()) - if value is None: - message.append(f"\t{name}: This property is not present in the data file.\n") - elif value.has_value(): + # A property the data file does not have needs the Enterprise data + # file, and its value is not read. + if not ExampleUtils.has_property(engine, name): + message.append(f"\t{name}: {ExampleUtils.NEEDS_ENTERPRISE}\n") + return True + value = ip_data.get(name.lower()) + if value.has_value(): message.append(f"\t{name}: {ExampleUtils.format_value(value.value())}\n") else: message.append(f"\t{name}: {value.no_value_message()}\n") + return False def main(argv): # Use the supplied path for the data file or find the data file in the # project space. - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ASN_DATAFILE_NAME) + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. logger = Logger(min_level="info") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py index 56ea9799..e6f25806 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/gettingstarted_web/app.py @@ -44,6 +44,10 @@ add_country_translation, build_country_options) from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-gettingstarted_web-app.py&utm_term=enterprise-data-file" + app = Flask(__name__) pipeline = None @@ -69,6 +73,12 @@ def get_value(ip_data, property): def country_dropdown(translated): + # Only the Enterprise data file has the country codes the dropdown is + # built from. + if not ExampleUtils.has_property( + pipeline.flow_elements_list["ip"], "CountryCodesGeographical"): + return (f"

    The country dropdown {ExampleUtils.NEEDS_ENTERPRISE}. " + f"Contact us for a copy.

    ") # The geographical "All" lists are index-aligned: option i is # (CountryCodesGeographicalAll[i], CountryNamesGeographicalAllTranslated[i]), diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py index 3819776c..f9e4a0fb 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/metadata_console.py @@ -25,7 +25,8 @@ # # This example shows how to list the properties available from the # on-premise IP intelligence engine, along with their types and -# descriptions. +# descriptions. By default it uses the free Lite data file, which it +# downloads when it is missing. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -34,7 +35,11 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=enterprise-data-file" class MetadataConsole(): @@ -59,9 +64,13 @@ def run(self, data_file, logger, output): if property_metadata.get("description"): output(f" {property_metadata['description']}") + if ExampleUtils.is_free_data_file(engine): + output("") + output(ExampleUtils.get_enterprise_offer(CONTACT_US)) + def main(argv): - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) logger = Logger(min_level="info") @@ -69,10 +78,11 @@ def main(argv): MetadataConsole().run(data_file, logger, print) else: logger.log("error", - "Failed to find an IP intelligence data file. Supply its path " - "as the first command line argument. To obtain an enterprise " - "data file for testing, please contact us: " - "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-metadata_console.py&utm_term=data-file-required") + "Failed to find an IP intelligence data file. Make sure the " + "data submodule is there, by running " + "`git submodule update --init --recursive`, and run this " + "example again to download the file, or supply the path to a " + "data file as the first command line argument.") if __name__ == "__main__": main(sys.argv[1:]) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py index 2f4f5e18..60fda04e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/mixed_console.py @@ -28,11 +28,15 @@ # User-Agent (used by device detection) and an IP address (used by IP # intelligence), and both engines process the flow data. # -# Two data files are required: +# Two data files are used, which may be supplied as the first two command +# line arguments: # - a Device Detection data file (.hash) # - an IP Intelligence data file (.ipi) # -# They may be supplied as the first two command line arguments. +# By default the free Lite data files are used, which are downloaded when +# they are missing, and each property only the Enterprise data files have is +# shown as needing them. Contact us for a copy: +# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-mixed_console.py&utm_term=header # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -46,7 +50,11 @@ from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import IpIntelligenceOnPremise from fiftyone_ipintelligence_examples.example_utils import ExampleUtils -from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME + +# Where to ask for a copy of the Enterprise data files, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-mixed_console.py&utm_term=enterprise-data-file" # Default name of the device detection data file (the lite file the data # repository links to). @@ -143,10 +151,15 @@ def run(self, device_file, ip_file, logger, output): # The pipeline is closed at the end of the with block, which closes # both engines and releases the native data they hold. with pipeline: + needs_enterprise = False for evidence in COMBINED_EVIDENCE: - self.analyse_evidence(evidence, pipeline, output) + needs_enterprise |= self.analyse_evidence( + evidence, pipeline, device_engine, ip_engine, output) + if needs_enterprise: + output("\n" + ExampleUtils.get_enterprise_note(CONTACT_US)) - def analyse_evidence(self, evidence, pipeline, output): + def analyse_evidence(self, evidence, pipeline, device_engine, ip_engine, + output): # The FlowData is closed at the end of the with block, which releases # the results of both engines straight away. @@ -165,36 +178,42 @@ def analyse_evidence(self, evidence, pipeline, output): data.process() message = [] + needs_enterprise = False # Device detection results, keyed under "device". message.append("Device Detection Results:\n") for name in DEVICE_PROPERTIES: - self.outputValue(data.device, name, message) + needs_enterprise |= self.outputValue( + device_engine, data.device, name, message) # IP intelligence results, keyed under "ip". message.append("\nIP Intelligence Results:\n") for name in IP_PROPERTIES: - self.outputValue(data.ip, name, message) + needs_enterprise |= self.outputValue( + ip_engine, data.ip, name, message) output("".join(message)) - - def outputValue(self, aspect_data, name, message): - # `get_value` returns None when the property is not present in the - # data file, rather than raising. - value = ExampleUtils.get_value(aspect_data, name.lower()) - if value is None: - message.append(f"\t{name}: This property is not present in the data file.\n") - elif value.has_value(): + return needs_enterprise + + def outputValue(self, engine, aspect_data, name, message): + # A property the engine's data file does not have needs the + # Enterprise data file, and its value is not read. + if not ExampleUtils.has_property(engine, name): + message.append(f"\t{name}: {ExampleUtils.NEEDS_ENTERPRISE}\n") + return True + value = aspect_data.get(name.lower()) + if value.has_value(): message.append(f"\t{name}: {ExampleUtils.format_value(value.value())}\n") else: message.append(f"\t{name}: {value.no_value_message()}\n") + return False def main(argv): # Use the supplied paths for the data files, or find the device detection # and IP intelligence files in the project space. device_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(HASH_DATAFILE_NAME) - ip_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(ASN_DATAFILE_NAME) + ip_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(LITE_DATAFILE_NAME) # Configure a logger to output to the console. logger = Logger(min_level="info") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py index 4785249e..a9e92fb9 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/offlineprocessing.py @@ -28,7 +28,9 @@ # # We create a pipeline using the on-premise engine, deserialize the # YAML evidence records, pass each entry to the pipeline and write the -# resolved properties out to another YAML file. +# resolved properties out to another YAML file. By default the free Lite +# data file is used, which is downloaded when it is missing, and Country, +# which only the Enterprise data file has, is recorded as needing it. # # Required PyPi Dependencies: # - fiftyone_ipintelligence @@ -42,12 +44,17 @@ from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import LITE_DATAFILE_NAME # Number of evidence records to process in this example. RECORD_LIMIT = 100 +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-offlineprocessing.py&utm_term=enterprise-data-file" + + class OfflineProcessing(): def run(self, data_file, evidence_yaml, logger, output): @@ -71,6 +78,7 @@ def run(self, data_file, evidence_yaml, logger, output): ExampleUtils.check_data_file(pipeline, logger) records = 0 + needs_enterprise = False with open(evidence_yaml, "r") as fin: for evidence in yaml.load_all(fin): if records >= RECORD_LIMIT: @@ -78,9 +86,14 @@ def run(self, data_file, evidence_yaml, logger, output): # write the yaml document separator output("---") - self.analyse_evidence(evidence, pipeline, output, yaml) + needs_enterprise |= self.analyse_evidence( + evidence, pipeline, output, yaml) records += 1 + if needs_enterprise: + # A comment keeps the output a valid YAML stream. + output("# " + ExampleUtils.get_enterprise_note(CONTACT_US)) + # write the yaml document end marker output("...") logger.log("info", f"Processed {records} evidence records") @@ -106,16 +119,20 @@ def analyse_evidence(self, evidence, pipeline, output, yaml): # Add the values that we want to store against the record. result["ip.registeredname"] = ExampleUtils.get_human_readable(ip_data, "registeredname") result["ip.registeredcountry"] = ExampleUtils.get_human_readable(ip_data, "registeredcountry") + # Only the Enterprise data file has Country. + result["ip.country"] = ExampleUtils.get_enterprise_value( + pipeline.flow_elements_list["ip"], ip_data, "country") buffer = io.BytesIO() yaml.dump(result, buffer) output(buffer.getvalue().decode("utf-8").rstrip()) + return result["ip.country"] == ExampleUtils.NEEDS_ENTERPRISE def main(argv): - # Use the supplied path for the data file or find the enterprise file - # in the project space. - data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + # Use the supplied path for the data file or find the free Lite file in + # the project space, which is downloaded when it is missing. + data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME) evidence_file = argv[1] if len(argv) > 1 else ExampleUtils.find_file(EVIDENCE_FILE_NAME) # Configure a logger to output to the console. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index 1d910d81..c76f930a 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -55,13 +55,17 @@ from fiftyone_ipintelligence.ipintelligence_pipelinebuilder import IpIntelligencePipelineBuilder from fiftyone_ipintelligence_examples.example_utils import ExampleUtils from fiftyone_pipeline_core.logger import Logger -from fiftyone_ipintelligence_shared.example_constants import ENTERPRISE_DATAFILE_NAME +from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME # The IP intelligence properties used to measure performance. Restricting the # work to a single component keeps the benchmark focused on lookup speed. PERFORMANCE_PROPERTY = "registeredcountry" +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_examples-src-fiftyone_ipintelligence_examples-onpremise-performance_console.py&utm_term=enterprise-data-file" + # The number of records processed before timing starts, so that any lazy # initialisation is not included in the timed run. WARM_UP_RECORDS = 100 @@ -214,7 +218,9 @@ def main(argv): help="Performance profile: MaxPerformance (default) or LowMemory") args = ap.parse_args(argv) - data_file = args.data_file or ExampleUtils.find_file(ENTERPRISE_DATAFILE_NAME) + # By default the free ASN data file is used, which is far smaller than the + # Lite file and is downloaded when it is missing. + data_file = args.data_file or ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = args.evidence_file or ExampleUtils.find_file(EVIDENCE_FILE_NAME) logger = Logger(min_level="info") @@ -227,6 +233,9 @@ def main(argv): if args.json_output: with open(args.json_output, "w") as fout: json.dump(results, fout) + if ExampleUtils.is_free_data_file_at(data_file): + print("") + print(ExampleUtils.get_enterprise_offer(CONTACT_US)) else: logger.log("error", "Failed to find a data file or evidence file. Make sure the " diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py new file mode 100644 index 00000000..e417bd79 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples_without_settings.py @@ -0,0 +1,133 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Runs the on-premise examples as they run from a fresh clone, with no data +# file setting and no licence key, so that they use the free data files. +# Each must finish, show the properties only the Enterprise data file has as +# needing it, and give the link to ask for a copy. + +import io +import os +import unittest +from contextlib import redirect_stdout +from unittest import mock + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils +from fiftyone_ipintelligence_examples.onpremise import compare_console +from fiftyone_ipintelligence_examples.onpremise import countries_console +from fiftyone_ipintelligence_examples.onpremise import datafileupdate_console +from fiftyone_ipintelligence_examples.onpremise import gettingstarted_console +from fiftyone_ipintelligence_examples.onpremise import metadata_console +from fiftyone_ipintelligence_examples.onpremise import mixed_console +from fiftyone_ipintelligence_examples.onpremise import offlineprocessing +from fiftyone_ipintelligence_examples.onpremise import performance_console + +# The settings the examples read, which a fresh clone does not have. +SETTINGS = ["license_key", "LICENSE_KEY", ExampleUtils.DATA_FILE_ENV_VAR] + +# Options that keep the performance example short. They leave the data file +# to the example. +PERFORMANCE_OPTIONS = ["--record_limit", "200", "--workers", "1"] + +# The mixed example combines IP intelligence with device detection, which +# is an optional dependency. +try: + import fiftyone_devicedetection_onpremise # noqa: F401 + device_detection_available = True +except ImportError: + device_detection_available = False + + +def run_without_settings(main, argv=None): + """Runs an example's main with the settings taken out of the + environment, and returns what it printed.""" + printed = io.StringIO() + with mock.patch.dict(os.environ), redirect_stdout(printed): + for name in SETTINGS: + os.environ.pop(name, None) + main(argv or []) + return printed.getvalue() + + +class ExamplesWithoutSettingsTests(unittest.TestCase): + + def test_gettingstarted_console(self): + printed = run_without_settings(gettingstarted_console.main) + self.assertIn("Country: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(gettingstarted_console.CONTACT_US), + printed) + + def test_countries_console(self): + printed = run_without_settings(countries_console.main) + self.assertIn( + "Listing the countries an IP address is most likely in needs the " + "CountryCodesGeographical and CountryCodesPopulation properties", + printed) + self.assertIn(countries_console.CONTACT_US, printed) + + def test_metadata_console(self): + printed = run_without_settings(metadata_console.main) + self.assertIn( + ExampleUtils.get_enterprise_offer(metadata_console.CONTACT_US), + printed) + + def test_offlineprocessing(self): + printed = run_without_settings(offlineprocessing.main) + self.assertIn("ip.country: " + ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + "# " + ExampleUtils.get_enterprise_note( + offlineprocessing.CONTACT_US), + printed) + + def test_compare_console(self): + printed = run_without_settings(compare_console.main) + self.assertIn( + "Comparing known locations with IP Intelligence needs the " + "Latitude, Longitude, LocationConfidence and Country properties", + printed) + self.assertIn(compare_console.CONTACT_US, printed) + + def test_performance_console(self): + printed = run_without_settings( + performance_console.main, PERFORMANCE_OPTIONS) + self.assertIn( + ExampleUtils.get_enterprise_offer(performance_console.CONTACT_US), + printed) + + def test_datafileupdate_console(self): + printed = run_without_settings(datafileupdate_console.main) + self.assertIn("The data file in use is the", printed) + self.assertIn(datafileupdate_console.CONTACT_US, printed) + + @unittest.skipUnless( + device_detection_available, "device detection is not installed") + def test_mixed_console(self): + printed = run_without_settings(mixed_console.main) + self.assertIn(ExampleUtils.NEEDS_ENTERPRISE, printed) + self.assertIn( + ExampleUtils.get_enterprise_note(mixed_console.CONTACT_US), + printed) + + +if __name__ == "__main__": + unittest.main() From 5ad5980fd2d192c53c9856edf3e2966b55864ba1 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 01:46:03 +0000 Subject: [PATCH 17/26] Build the released packages when a nightly publish has nothing to release (#45) --- ci/build-package.ps1 | 7 +++++-- ci/packages.Tests.ps1 | 30 ++++++++++++++++++++++++++++++ ci/packages.ps1 | 26 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index f806b1f7..057d3167 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -6,9 +6,12 @@ $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'packages.ps1') # Builds the packages this publish releases, each at its own version from the -# release plan. $Version is common-ci's label for the run and is not used. +# release plan. When nothing is to be released, every package is built at its +# released version, so that the tests have packages to install and the nightly +# publish passes without publishing anything. $Version is common-ci's label for +# the run and is not used. -$plan = @(Get-ReleasePlan $RepoName) +$plan = @(Get-BuildPlan $RepoName) # fiftyone_pipeline_did carries the OWID source inside itself, copied in by the # pipeline product's script; see pipeline/ci/copy-owid-source.ps1. diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 2a2887d9..766d1380 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -249,6 +249,36 @@ Describe 'Get-ReleasePlan' { } } +Describe 'Get-BuildPlan' { + It 'is the release plan when a package is to be released' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('pipeline/engines/a.py') + $plan = @(Get-BuildPlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('engines', 'dd', 'ipi') + ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39' + } + + It 'builds every published package at its released version when nothing changed' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @() + @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 + $plan = @(Get-BuildPlan $script:Repo $script:Graph) + $plan.Name | Should -Be @('core', 'did', 'engines', 'location', 'dd', 'ipi') + ($plan | Where-Object Name -eq 'core').Version | Should -Be '4.5.38' + ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.96' + ($plan | Where-Object Name -eq 'location').Version | Should -Be '4.5.4' + ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0' + } + + It 'builds the released versions from a commit already released' { + $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + Add-Commit $script:Repo @('device/dd/a.py') + git -C $script:Repo tag 'dd/4.5.97' + ($plan = @(Get-BuildPlan $script:Repo $script:Graph)).Count | Should -Be 6 + ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' + } +} + Describe 'the real packages.json' { BeforeAll { $script:Real = Get-Packages (Split-Path $PSScriptRoot) diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 70e6af0c..429c43ff 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -292,3 +292,29 @@ function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { } } } + +# What a build of HEAD makes: the release plan, or, when nothing changed since +# the last release, every published package at its released version. Nothing +# is published then, because package-update-required.ps1 finds the release +# plan empty, but the publish's tests still have the packages to install. +function Get-BuildPlan([string]$RepoPath, $Packages = $null) { + if ($null -eq $Packages) { + $Packages = Get-Packages $RepoPath + } + $plan = @(Get-ReleasePlan $RepoPath $Packages) + if ($plan.Count -gt 0) { + return $plan + } + $published = @($Packages | Where-Object { $_.Publish -eq $true }) + foreach ($name in (Sort-ByDependency $published @($published.Name))) { + $package = $published | Where-Object { $_.Name -eq $name } + $last = @(Get-ReleasedVersions $RepoPath $name)[-1] + [pscustomobject]@{ + Name = $name + Path = $package.Path + Version = "$last" + Previous = $last + Reason = 'already released, built for the tests' + } + } +} From 0507eefa25242e0c9080b0766725cb1650b7e190 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 7 Nov 2026 02:44:45 +0000 Subject: [PATCH 18/26] Share only the evidence usage sharing accepts unless asked for all (#50) --- .../src/fiftyone_pipeline_core/evidence.py | 3 +- .../fiftyone_pipeline_core/tests/test_core.py | 18 ++ .../examples/usagesharing/51d.json | 4 +- .../share_usage.py | 68 ++-- .../share_usage_evidencekeyfilter.py | 161 ++++++---- .../test_share_usage_evidencekeyfilter.py | 168 ++++++++++ .../tests/test_usage.py | 296 ++++++++++++++++++ 7 files changed, 635 insertions(+), 83 deletions(-) create mode 100644 pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py index 627ca7d8..f7067647 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/evidence.py @@ -98,7 +98,8 @@ def filter(self) -> None: if self.evidence_filtered: return - for key in self.evidence: + # Loop over a copy of the keys, as keys are removed on the way. + for key in list(self.evidence): if not any([ flow_element.filter_evidence_key(key) for flow_element diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py index f26acde6..21407b33 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_core.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py @@ -58,6 +58,24 @@ def testEvidenceKeyFilter(self): nullEvidence = testPipeline.flowdata.evidence.get("header.other-evidence") self.assertTrue(nullEvidence == None) + # Test that evidence no element accepts is removed without an error. + # The JavaScript and set header elements accept every key, so they are + # left out. + def testEvidenceKeyFilterRemovesKeys(self): + + pipeline = (PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}))\ + .add(ExampleFlowElement1())\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "test") + flowdata.evidence.add("header.accept-encoding", "gzip") + flowdata.evidence.add("header.host", "example.com") + flowdata.process() + self.assertEqual( + flowdata.evidence.get_all(), {"header.user-agent": "test"}) + # Test Getter methods def testGet(self): diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json index 90cfbada..7196a8cb 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/examples/usagesharing/51d.json @@ -9,8 +9,10 @@ "interval": 1200, "requested_package_size": 50, "cookie": "", + "track_session": false, "query_whitelist": [], - "header_blacklist": [] + "header_blacklist": [], + "share_all": false } } ] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py index 3173318e..210dfb0a 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage.py @@ -31,7 +31,8 @@ import requests from fiftyone_pipeline_engines.engine import Engine -from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter +from .share_usage_evidencekeyfilter import ShareUsageEvidenceKeyFilter, \ + ShareUsageTrackerEvidenceKeyFilter from .share_usage_tracker import ShareUsageTracker # The maximum length of a piece of evidence's value which can be @@ -54,7 +55,9 @@ def __init__( header_blacklist = [], share_percentage = 1, endpoint = "https://devices-v4.51degrees.com/new.ashx", - request_timeout=REQUEST_TIMEOUT): + request_timeout=REQUEST_TIMEOUT, + share_all = False, + track_session = False): """! Constructor for ShareUsage element @@ -70,22 +73,32 @@ def __init__( reduce the overhead of sending HTTP messages. For example, the 51Degrees cloud service uses a value of 2500. @type cookie: string - @param cookie: If a cookie is being used to identify user - sessions, it can be specified here in order to reduce the - sharing of duplicated data. + @param cookie: The name of the cookie that holds the session id. + It has no effect unless track_session is True. The cookie itself + is not shared unless share_all is True. + @type track_session: bool + @param track_session: If True, the cookie named by cookie is used + to tell user sessions apart, so evidence already shared is shared + again when it comes from a new session. False by default. It + needs cookie to be set, and has no effect when share_all is True, + because every cookie is then shared and used to find repeats. @type query_whitelist: list - @param query_whitelist: By default query string and HTTP form + @param query_whitelist: By default query string and HTTP form parameters are not shared unless prefixed with '51D_'. - If you need to share query string parameters, a list can be - specified here. - @type query_blacklist: list - @param query_blacklist: By default, all HTTP headers - (except a few, such as Cookies) are shared. - Individual headers can be excluded from sharing by adding them + If you need to share query string parameters, a list can be + specified here. None shares every parameter. + @type header_blacklist: list + @param header_blacklist: By default, all HTTP headers + except Cookie are shared. + Individual headers can be excluded from sharing by adding them to this list. @type share_percentage : float @param share_percentage: approximate proportion of requests to be shared. 1 = 100%, 0.5 = 50%, etc.. + @type share_all: bool + @param share_all: If True, all evidence is shared apart from the + email address, the salt used for hashed email identifiers, and the + request's query string and path. False by default. """ super(ShareUsage, self).__init__() @@ -102,6 +115,25 @@ def __init__( self.share_percentage = share_percentage self.endpoint = endpoint self.request_timeout = request_timeout + self.share_all = share_all + self.track_session = track_session + + # The session id and sequence number are kept in evidence, as the + # usage data carries them separately. + included = None if query_whitelist is None else \ + list(query_whitelist) + ["session-id", "sequence"] + + # When sessions are tracked, the pipeline keeps the session cookie + # for the tracker. The data sent leaves it out unless all evidence + # is shared. + self.evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, share_all) + self.data_evidence_key_filter = ShareUsageEvidenceKeyFilter( + cookie, included, header_blacklist, False, share_all) + self.tracker_evidence_key_filter = \ + ShareUsageTrackerEvidenceKeyFilter( + cookie, included, header_blacklist, track_session, + share_all) # Add the share usage tracker which detects when # to send up sharing data @@ -188,11 +220,7 @@ def get_evidence_key_filter(self): """ - return ShareUsageEvidenceKeyFilter( - cookie = self.cookie, - query_whitelist = self.query_whitelist, - header_blacklist = self.header_blacklist - ) + return self.evidence_key_filter def share_send_usage(self): """! @@ -291,7 +319,7 @@ def process_internal(self, flow_data): """ if random.uniform(0, 1) <= self.share_percentage: - cachekey = self.get_evidence_key_filter().filter_evidence(flow_data.evidence.get_all()) + cachekey = self.tracker_evidence_key_filter.filter_evidence(flow_data.evidence.get_all()) cachekey = json.dumps(cachekey) share = self.tracker.track(cachekey) if share: @@ -327,7 +355,7 @@ def get_data_from_evidence(self, flow_data): f"The value '{value}' could not be parsed to an integer.") else: # Check if we can send this piece of evidence - if self.get_evidence_key_filter().filter_evidence_key(key): + if self.data_evidence_key_filter.filter_evidence_key(key): data.try_add(key, value) return data @@ -394,7 +422,7 @@ def try_add(self, key, value): category = "" field = key - first_separator = key.index(".") + first_separator = key.find(".") if first_separator > 0: category = key[0:first_separator] field = key[first_separator + 1:len(key)] diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py index a9d0d2ac..5f2292f3 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/src/fiftyone_pipeline_engines_fiftyone/share_usage_evidencekeyfilter.py @@ -22,41 +22,64 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +# Cookies, session values and query string parameters whose names start with +# this are the ones 51Degrees sets, and are shared. +FIFTYONE_PREFIX = "51d_" + +# The Cookie header carries every cookie, including the ones the cookie rule +# leaves out, so it is not shared unless all evidence is. +BLOCKED_HEADERS = ("cookie",) + +# Evidence that is never shared, whatever the settings, matched as the whole +# key or after a '.'. 'id.email' and 'id.salt' are a raw email address and the +# salt used to make hashed email identifiers. 'request-query' is the whole query +# string, which would undo the rules for its parameters. 'request-path' would +# make each page a visitor opens look like a new session to the tracker. +NEVER_SHARED_KEYS = ("id.email", "id.salt", "request-query", "request-path") + class ShareUsageEvidenceKeyFilter(BasicListEvidenceKeyFilter): """ - The ShareUsageEvidenceKeyFilter filters out all evidence - not needed by the 51Degrees ShareUsage service. - It allows for a specific whitelist of query strings, - a blacklist of headers and a specific cookie used for - session information + The ShareUsageEvidenceKeyFilter decides which evidence the 51Degrees + ShareUsage service is sent. Headers are shared unless they are blocked, + cookies and session values only when their names start with 51D_, and + query string and form parameters only when their names start with 51D_ or + are on the list to share. Any other evidence is shared. """ - def __init__(self, cookie = None, query_whitelist = [], header_blacklist = []): + def __init__( + self, + cookie = None, + query_whitelist = (), + header_blacklist = (), + include_session = False, + share_all = False): """! Constructor for ShareUsageEvidenceKeyFilter @type cookie: string - @param cookie: which cookie is used to track evidence + @param cookie: the name of the cookie that identifies the session @type query_whitelist: list - @param query_whitelist: list of query string whitelist evidence to keep - @type query_blacklist: list - @param query_blacklist: list of header evidence to exclude - @type - @param + @param query_whitelist: query string and form parameters to share as + well as those starting 51D_, or None to share every parameter + @type header_blacklist: list + @param header_blacklist: headers not to share + @type include_session: bool + @param include_session: whether the session cookie is included + @type share_all: bool + @param share_all: whether all evidence is shared, apart from the + evidence that is never shared """ - self.query_whitelist = query_whitelist + self.cookie = cookie.lower() if cookie else None - self.header_blacklist = header_blacklist + self.query_whitelist = None if query_whitelist is None else \ + [parameter.lower() for parameter in query_whitelist] - self.cookie = cookie + self.header_blacklist = \ + [header.lower() for header in header_blacklist or ()] - if cookie is None and \ - (query_whitelist is None or len(query_whitelist) == 0) and \ - (header_blacklist is None or len(header_blacklist) == 0): - self.share_all = True - else: - self.share_all = False + self.include_session = include_session + + self.share_all = share_all - def filter_evidence_key(self, key): """! @@ -69,42 +92,58 @@ def filter_evidence_key(self, key): @return: Is this key in the filter's keys list? """ - # get prefix and key of evidence - key_parts = key.lower().split(".") - - prefix = key_parts[0] - suffix = key_parts[1] - - result = self.share_all - - if not self.share_all: - if prefix == "header": - # Add the header to the list if the header name does not - # appear in the list of blocked headers - result = suffix not in self.header_blacklist - elif prefix == "cookie": - # Only add cookies that start with the 51Degrees cookie - # prefix - result = suffix.startswith("51d_") or \ - (self.include_session and suffix == self.cookie) - elif prefix == "session": - # Only session values that start with the 51Degrees - # cookie prefix - result = suffix.startswith("51d_") - elif prefix == "query": - # If no query string parameter filter was specified - # then share all of them. - # Otherwise, only include query string parameters that - # start with 51d_ or that have been specified in - # the constructor. - result = self.query_whitelist is None or \ - len(self.query_whitelist) == 0 or \ - suffix.startswith("51d_") or \ - suffix in self.query_whitelist - else: - # Add anything that is not a cookie, header, session - # variable or query parameter - result = True - - return result - \ No newline at end of file + key = key.lower() + + if any(key == never_shared or key.endswith("." + never_shared) + for never_shared in NEVER_SHARED_KEYS): + return False + + if self.share_all: + return True + + # The prefix is everything before the first dot and the field + # everything after it, so a field can itself contain dots. + prefix, separator, field = key.partition(".") + + if not separator: + return True + elif prefix == "header": + # Add the header to the list if the header name does not + # appear in the list of blocked headers + return field not in self.header_blacklist and \ + field not in BLOCKED_HEADERS + elif prefix == "cookie": + # Only add cookies that start with the 51Degrees cookie + # prefix + return field.startswith(FIFTYONE_PREFIX) or \ + (self.include_session and field == self.cookie) + elif prefix == "session": + # Only session values that start with the 51Degrees + # cookie prefix + return field.startswith(FIFTYONE_PREFIX) + elif prefix == "query": + # If no query string parameter list was given then share all + # of them. Otherwise, only include query string parameters + # that start with 51d_ or that are in the list. + return self.query_whitelist is None or \ + field.startswith(FIFTYONE_PREFIX) or \ + field in self.query_whitelist + else: + # Add anything that is not a cookie, header, session + # variable or query parameter + return True + +class ShareUsageTrackerEvidenceKeyFilter(ShareUsageEvidenceKeyFilter): + """ + The evidence the ShareUsageTracker compares to decide whether a request + repeats one already shared. The session id and sequence number are left + out, because they make every request different. + """ + + def filter_evidence_key(self, key): + + if key.lower() in ("query.session-id", "query.sequence"): + return False + + return super(ShareUsageTrackerEvidenceKeyFilter, self)\ + .filter_evidence_key(key) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py new file mode 100644 index 00000000..16184d52 --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_share_usage_evidencekeyfilter.py @@ -0,0 +1,168 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_engines_fiftyone.share_usage_evidencekeyfilter import \ + ShareUsageEvidenceKeyFilter + + +class ShareUsageEvidenceKeyFilterTests(unittest.TestCase): + + def test_default_rules(self): + """! + Check that by default headers other than Cookie are shared, cookies, + session values and query string parameters only when their names + start with 51D_, and any other evidence. + """ + + keys = ShareUsageEvidenceKeyFilter() + + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + self.assertFalse(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.51d_screenpixelswidth")) + self.assertTrue(keys.filter_evidence_key("cookie.51D_ScreenPixelsWidth")) + self.assertFalse(keys.filter_evidence_key("cookie.sessionid")) + self.assertFalse(keys.filter_evidence_key("cookie.a.b")) + self.assertTrue(keys.filter_evidence_key("session.51d_value")) + self.assertFalse(keys.filter_evidence_key("session.org.example.attribute")) + self.assertTrue(keys.filter_evidence_key("query.51d_gethighentropyvalues")) + self.assertFalse(keys.filter_evidence_key("query.password")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertTrue(keys.filter_evidence_key("server.client-ip")) + self.assertTrue(keys.filter_evidence_key("nodot")) + + def test_field_is_everything_after_the_first_dot(self): + """! + Check that a name containing a dot is matched as a whole against + the parameters to share. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertFalse(keys.filter_evidence_key("query.valid")) + self.assertFalse(keys.filter_evidence_key("query.other.name")) + + def test_lists_ignore_case(self): + """! + Check that the headers not to share and the parameters to share + are matched without regard to case. + """ + + keys = ShareUsageEvidenceKeyFilter( + query_whitelist=["Campaign"], + header_blacklist=["X-Secret"]) + + self.assertFalse(keys.filter_evidence_key("header.x-secret")) + self.assertTrue(keys.filter_evidence_key("query.campaign")) + self.assertTrue(keys.filter_evidence_key("header.user-agent")) + + def test_no_parameter_list_shares_every_parameter(self): + """! + Check that None in place of the parameters to share shares every + parameter, and an empty list only those starting 51D_. + """ + + every = ShareUsageEvidenceKeyFilter(query_whitelist=None) + prefixed = ShareUsageEvidenceKeyFilter(query_whitelist=[]) + + self.assertTrue(every.filter_evidence_key("query.campaign")) + self.assertFalse(prefixed.filter_evidence_key("query.campaign")) + self.assertTrue(prefixed.filter_evidence_key("query.51d_campaign")) + + def test_session_cookie(self): + """! + Check that the session cookie is included only when asked for, and + that no other ordinary cookie is. + """ + + kept = ShareUsageEvidenceKeyFilter(cookie="SessionId", include_session=True) + sent = ShareUsageEvidenceKeyFilter(cookie="SessionId") + + self.assertTrue(kept.filter_evidence_key("cookie.sessionid")) + self.assertFalse(sent.filter_evidence_key("cookie.sessionid")) + self.assertFalse(kept.filter_evidence_key("cookie._ga")) + self.assertFalse(sent.filter_evidence_key("cookie._ga")) + + def test_share_all(self): + """! + Check that sharing everything shares the evidence the other rules + leave out, apart from the evidence that is never shared. + """ + + keys = ShareUsageEvidenceKeyFilter(share_all=True) + + self.assertTrue(keys.filter_evidence_key("header.cookie")) + self.assertTrue(keys.filter_evidence_key("cookie.sessionid")) + self.assertTrue(keys.filter_evidence_key("query.password")) + self.assertTrue(keys.filter_evidence_key("query.id.usage")) + self.assertTrue(keys.filter_evidence_key("server.request-method")) + self.assertFalse(keys.filter_evidence_key("query.id.email")) + self.assertFalse(keys.filter_evidence_key("header.id.email")) + self.assertFalse(keys.filter_evidence_key("query.id.salt")) + self.assertFalse(keys.filter_evidence_key("header.id.salt")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) + self.assertFalse(keys.filter_evidence_key("server.request-path")) + + def test_never_shared_in_every_mode(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are never shared, + whether bare or after a prefix, and without regard to case. + """ + + for keys in ( + ShareUsageEvidenceKeyFilter(), + ShareUsageEvidenceKeyFilter(query_whitelist=None), + ShareUsageEvidenceKeyFilter(share_all=True)): + for key in ( + "id.email", "id.salt", + "query.id.email", "QUERY.ID.EMAIL", + "header.id.salt", "HEADER.ID.SALT", + "server.request-query", "SERVER.REQUEST-QUERY", + "server.request-path", "SERVER.REQUEST-PATH"): + self.assertFalse(keys.filter_evidence_key(key), key) + + def test_never_shared_keys_match_whole_parts(self): + """! + Check that a key that only ends with the same letters as a key + never shared is treated as an ordinary key. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["valid.email"]) + + self.assertTrue(keys.filter_evidence_key("query.valid.email")) + self.assertTrue( + ShareUsageEvidenceKeyFilter(share_all=True) + .filter_evidence_key("query.valid.email")) + + def test_naming_a_parameter_does_not_share_the_query_string(self): + """! + Check that sharing one parameter does not share the whole query + string, which carries every parameter. + """ + + keys = ShareUsageEvidenceKeyFilter(query_whitelist=["utm_source"]) + + self.assertTrue(keys.filter_evidence_key("query.utm_source")) + self.assertFalse(keys.filter_evidence_key("server.request-query")) diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py index 8329645b..4f1bea40 100644 --- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py +++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py @@ -431,6 +431,302 @@ def test_includes_header(self): self.assertIn('&', app.received[0]) + def test_default_shares_only_accepted_evidence(self): + """! + Check that with no options the element shares headers and the + 51Degrees cookies and parameters, and leaves out other cookies, the + Cookie header, and other query string and form parameters. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.51d_screenpixelswidth", "1080") + flowdata.evidence.add("query.51d_gethighentropyvalues", "hev") + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("header.cookie", "sessionid=header value") + flowdata.evidence.add("query.password", "password value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('
    some user agent
    ', app.received[0]) + self.assertIn('1080', app.received[0]) + self.assertIn('hev', app.received[0]) + self.assertNotIn("cookie value", app.received[0]) + self.assertNotIn("header value", app.received[0]) + self.assertNotIn("password value", app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_never_shared_evidence(self): + """! + Check that the email address, the salt for hashed email + identifiers, the query string and the path are not shared, though + the rules for their prefixes would share them. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("id.email", "email value") + flowdata.evidence.add("header.id.salt", "salt value") + flowdata.evidence.add("server.request-query", "query string value") + flowdata.evidence.add("server.request-path", "path value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("email value", app.received[0]) + self.assertNotIn("salt value", app.received[0]) + self.assertNotIn("query string value", app.received[0]) + self.assertNotIn("path value", app.received[0]) + + def test_share_all(self): + """! + Check that sharing all evidence shares what the other rules leave + out, but not the evidence that is never shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + share_all=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("cookie.sessionid", "cookie value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.evidence.add("query.id.email", "email value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('cookie value', app.received[0]) + self.assertIn('campaign value', app.received[0]) + self.assertNotIn("email value", app.received[0]) + + def test_parameter_names_with_dots(self): + """! + Check that a query string parameter whose name contains a dot is + matched by its whole name against the parameters to share. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=["valid.email"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.valid.email", "valid value") + flowdata.evidence.add("query.valid", "first part value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn('valid value', app.received[0]) + self.assertNotIn("first part value", app.received[0]) + + def test_empty_query_whitelist(self): + """! + Check that an empty list of parameters to share shares only those + starting 51D_, and that None shares every parameter. + """ + + for query_whitelist, shared in (([], False), (None, True)): + app.received = [] + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + query_whitelist=query_whitelist, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("query.51d_param", "fiftyone value") + flowdata.evidence.add("query.campaign", "campaign value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("fiftyone value", app.received[0]) + self.assertEqual("campaign value" in app.received[0], shared) + + def test_session_cookie(self): + """! + Check that with session tracking on the session cookie is kept in + the evidence, but neither it nor any other ordinary cookie is + shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("cookie.sessionid", "session value") + flowdata.evidence.add("cookie._ga", "analytics value") + flowdata.process() + + self.waitForUsageThreads() + self.assertTrue(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("session value", app.received[0]) + self.assertNotIn("analytics value", app.received[0]) + + def test_session_cookie_without_session_tracking(self): + """! + Check that session tracking is off by default, and that naming the + session cookie then has no effect. The cookie is not kept, and + evidence already shared is not shared again for a new session. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid") + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertNotIn("session value", app.received[0]) + self.assertFalse(usage.get_evidence_key_filter().filter_evidence_key("cookie.sessionid")) + self.assertFalse(usage.track_session) + + def test_session_tracking(self): + """! + Check that with session tracking on, evidence already shared is + shared again for a new session but not for a repeat in the same + session, and that the session cookie is not shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + cookie="sessionid", + track_session=True) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session in ("session value 1", "session value 1", "session value 2"): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("cookie.sessionid", session) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 2) + received = "\n".join(app.received) + self.assertEqual(received.count("repeated user agent"), 2) + self.assertNotIn("session value", received) + + def test_ordinary_cookie_with_header_blacklist(self): + """! + Check that an ordinary cookie is left out without an error when + some headers are not to be shared. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint, + header_blacklist=["x-secret"]) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "some user agent") + flowdata.evidence.add("header.x-secret", "secret value") + flowdata.evidence.add("cookie.csrftoken", "token value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("some user agent", app.received[0]) + self.assertNotIn("secret value", app.received[0]) + self.assertNotIn("token value", app.received[0]) + + def test_repeat_with_new_sequence_not_shared(self): + """! + Check that a request differing from one already shared only in + its session id and sequence number is not shared again. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + for session, sequence in (("session 1", "1"), ("session 2", "2")): + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "repeated user agent") + flowdata.evidence.add("query.session-id", session) + flowdata.evidence.add("query.sequence", sequence) + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("session 1", app.received[0]) + + def test_evidence_without_a_prefix(self): + """! + Check that evidence whose key has no prefix is shared without an + error. + """ + + usage = ShareUsage( + requested_package_size=1, + endpoint=testEndpoint) + + pipeline = (PipelineBuilder())\ + .add(usage)\ + .build() + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("nodot", "no prefix value") + flowdata.process() + + self.waitForUsageThreads() + self.assertEqual(len(app.received), 1) + self.assertIn("no prefix value", app.received[0]) + # zzz is to make this test run last so no other tests depend on the receiver anymore # we depend on python unittests ordering tests alphabetically, which is not ideal, but works for now def test_zzz_timeout(self): From 8ba66c7022fa36c7ce246e9a2c4a9d4e3019c50e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 05:40:44 +0000 Subject: [PATCH 19/26] Fail the cloud tests when no resource key is set, and run the Cloud 51Did workflow (#58) * Fail the cloud tests when no resource key is set, and run the Cloud 51Did workflow Tests that exist to call the live cloud service skipped when no resource key reached them, and the 51Did live tests also skipped when the answer lacked what they checked, so a run with no key or with a key lacking a product reported success without the cloud being called. - Each test folder that calls the live service has the same live_cloud.py, which reads _51DEGREES_RESOURCE_KEY, then _51DEGREES_RESOURCE_KEY_BESPOKE, then resource_key, and gives the failure messages. Every skip for a missing key in the 51Did, Device Detection cloud, Device Detection example, IP Intelligence example and pipeline example tests is now a failure naming the variables, the product the key must carry and how to see what a key carries. - Where the 51Did answer lacks an identifier, the failure gives the cloud's null reason and the product. A host without the creator context fails. Without a licence key the redeem tests check the sealed result redeems as unreadable. - test_did_tier_live.py checks what a Basic key gets, being a 51Did for non-marketing and a refusal naming CloudV5FODiD for standard and personalized. - The live tests are marked, the unit steps deselect them through PYTEST_ADDOPTS, and the pipeline integration step now runs the 51Did live tests. The integration scripts fail when no TestResourceKey is passed, and the Device Detection one keeps the cloud package in the run when 51Degrees.csv is missing. - cloud-51did.yml moves to the root .github/workflows, where GitHub reads it, with its paths corrected. It fails when no key reaches it, and runs the Basic key only through the tier test. The licence-key tests in the pipeline examples and test_licensekey_live.py still skip, because CI has no cloud licence key yet (#13). * Wrap the tox.ini comment about the cloud test keys --- .../workflows/cloud-51did.yml | 47 +++-- device-detection/ci/run-integration-tests.ps1 | 13 +- device-detection/ci/run-unit-tests.ps1 | 21 +- .../tests/conftest.py | 32 +++ .../tests/live_cloud.py | 88 ++++++++ .../tests/test_cloud.py | 33 +-- .../tests/test_properties.py | 95 ++++----- .../fiftyone_devicedetection_cloud/tox.ini | 4 + .../readme.md | 5 +- .../tests/live_cloud.py | 88 ++++++++ .../tests/test_cloudexamples.py | 16 +- .../fiftyone_devicedetection_examples/tox.ini | 1 + ip-intelligence/ci/run-integration-tests.ps1 | 19 +- .../tests/live_cloud.py | 88 ++++++++ .../tests/test_examples.py | 28 ++- .../fiftyone_ipintelligence_examples/tox.ini | 2 + location/ci/run-integration-tests.ps1 | 5 +- pipeline/ci/run-integration-tests.ps1 | 24 ++- pipeline/ci/run-unit-tests.ps1 | 13 +- .../examples/readme.md | 6 +- .../examples/tests/helpers.py | 30 ++- .../examples/tests/live_cloud.py | 88 ++++++++ .../tests/test_cloud_mixed_console.py | 4 +- .../examples/tests/test_cloud_mixed_web.py | 4 +- .../examples/tox.ini | 15 +- .../fiftyone_pipeline_did/tests/conftest.py | 38 ++++ .../fiftyone_pipeline_did/tests/live_cloud.py | 88 ++++++++ .../tests/test_did_client_live.py | 188 +++++++++--------- .../tests/test_did_tier_live.py | 113 +++++++++++ pipeline/fiftyone_pipeline_did/tox.ini | 6 +- 30 files changed, 950 insertions(+), 252 deletions(-) rename {pipeline/.github => .github}/workflows/cloud-51did.yml (65%) create mode 100644 device-detection/fiftyone_devicedetection_cloud/tests/conftest.py create mode 100644 device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py create mode 100644 device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/live_cloud.py create mode 100644 pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py diff --git a/pipeline/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml similarity index 65% rename from pipeline/.github/workflows/cloud-51did.yml rename to .github/workflows/cloud-51did.yml index 95b128b4..5f258dc1 100644 --- a/pipeline/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -1,9 +1,12 @@ name: Cloud 51Did Tests # Runs the live cloud 51Did integration test -# (fiftyone_pipeline_did/tests/test_did_client_live.py) once for every -# _51DEGREES_RESOURCE_KEY* key passed in below. Each run sets the single -# _51DEGREES_RESOURCE_KEY the test reads. +# (pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py) once for +# every _51DEGREES_RESOURCE_KEY* key passed to its step below. Each run sets +# the single _51DEGREES_RESOURCE_KEY the test reads first. Those keys carry +# the FODid product and the CloudV5FODiD product the marketing usages need. +# The Basic key carries FODid without CloudV5FODiD, so it runs only the +# tests of what the Basic tier is entitled to (test_did_tier_live.py). # # The secrets are named one by one rather than passed as toJSON(secrets). # GitHub marks a workflow that hands every secret to a step as possibly @@ -11,10 +14,9 @@ name: Cloud 51Did Tests # A new resource key therefore needs a line here as well as the organisation # secret. # -# The test skips itself when no key is set, so a run with no secrets is green -# and proves nothing. That is deliberate, because a pull request from a fork -# has no secrets, and it is the reason the loop below reports which key each -# run used. +# A run with no key fails, because a cloud test that passes without calling +# the cloud has proved nothing. A pull request from a fork gets no secrets, +# so run this workflow from a branch in this repository. on: workflow_dispatch: @@ -29,8 +31,8 @@ on: pull_request: branches: [main] paths: - - 'fiftyone_pipeline_did/**' - - 'owid-python/**' + - 'pipeline/fiftyone_pipeline_did/**' + - 'pipeline/owid-python/**' - '.github/workflows/cloud-51did.yml' jobs: @@ -39,7 +41,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: fiftyone_pipeline_did + working-directory: pipeline/fiftyone_pipeline_did steps: - uses: actions/checkout@v4 with: @@ -56,26 +58,27 @@ jobs: # runs. shell: pwsh working-directory: . - run: ./ci/copy-owid-source.ps1 + run: ./pipeline/ci/copy-owid-source.ps1 - name: Install tox run: python -m pip install --upgrade pip tox - name: Cloud 51Did test for each resource key secret shell: pwsh env: - _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} _51DEGREES_RESOURCE_KEY_BESPOKE: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} _51DEGREES_RESOURCE_KEY_51DID: ${{ vars._51DEGREES_RESOURCE_KEY_51DID }} - # The redemption tests skip without a license key, and the service - # only opens a sealed result for a license key of the account whose - # resource key sealed it. So each license key is passed only with - # the resource key of the same suffix. + # The redemption tests check a sealed result reads back only with a + # license key, and the service only opens a sealed result for a + # license key of the account whose resource key sealed it. So each + # license key is passed only with the resource key of the same + # suffix, and a resource key without one checks that the result + # redeems as unreadable. _51DEGREES_LICENSE_KEY_51DID: ${{ secrets._51DEGREES_LICENSE_KEY_51DID }} run: | $keys = Get-ChildItem env: | Where-Object { $_.Name -like '_51DEGREES_RESOURCE_KEY*' -and $_.Value } if (-not $keys) { - Write-Host 'No _51DEGREES_RESOURCE_KEY* secrets available; skipping.' - exit 0 + Write-Host '::error::No _51DEGREES_RESOURCE_KEY* secret reached this job. The tests need a key that carries the FODid product, and the CloudV5FODiD product for the marketing usages.' + exit 1 } $failed = $false foreach ($k in $keys) { @@ -84,10 +87,14 @@ jobs: $licenseName = $k.Name -replace '^_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY' $env:_51DEGREES_LICENSE_KEY = [Environment]::GetEnvironmentVariable($licenseName) Write-Host "License key: $(if ($env:_51DEGREES_LICENSE_KEY) { $licenseName } else { 'none' })" - # -rs prints why each skipped test skipped, which tells a run that - # lacked a key apart from one the service turned away. + # -rs lists any test that skipped, and none should. tox -- tests/test_did_client_live.py -v -rs if ($LASTEXITCODE -ne 0) { $failed = $true } Write-Host '::endgroup::' } if ($failed) { exit 1 } + - name: Cloud 51Did tier test with the Basic key + if: ${{ !cancelled() }} + env: + _51DEGREES_RESOURCE_KEY_BASIC: ${{ vars._51DEGREES_RESOURCE_KEY_BASIC }} + run: tox -- tests/test_did_tier_live.py -v -rs diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index f1483528..c2d33cc4 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -14,11 +14,14 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run." - return -} elseif (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so cloud tests will not run." - $packages = "fiftyone_devicedetection_examples" + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." +} +# The cloud package runs without the CSV file too, because only its property +# coverage tests read the file, and they say so when it is missing. +if (!(Test-Path $RepoName/fiftyone_devicedetection_cloud/tests/51Degrees.csv)) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No CSV File::CSV file wasn't found, so the cloud property coverage tests will not run." } # nightly-publish-main workflow doesn't create the examples package, so diff --git a/device-detection/ci/run-unit-tests.ps1 b/device-detection/ci/run-unit-tests.ps1 index 7bb9dea9..4acebce4 100644 --- a/device-detection/ci/run-unit-tests.ps1 +++ b/device-detection/ci/run-unit-tests.ps1 @@ -9,11 +9,20 @@ if ($IsWindows) { Write-Output $env:TEMP } -# The cloud package's tests run against fixed cloud responses, so they need -# no resource key, no licence key and no network connection. Running them -# here means a broken cloud example shows as red on every build, rather than -# only when the integration tests happen to have a key to run with. -$packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise" -./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +# The cloud package's tests against fixed cloud responses need no resource +# key, no licence key and no network connection. Running them here means a +# broken cloud example shows as red on every build, rather than only when the +# integration tests happen to have a key to run with. Its tests marked live +# call the cloud service, and no key reaches this step, so they are left to +# run-integration-tests.ps1. Only a package whose tox.ini passes +# PYTEST_ADDOPTS sees the selection. +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "not live"' +try { + $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_onpremise" + ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py b/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py new file mode 100644 index 00000000..8890cf53 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker that keeps the live cloud tests out of the unit test run, which +# device-detection/ci/run-unit-tests.ps1 deselects. The tests against fixed +# cloud responses carry no marker, so they run in both steps. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index c5369df1..177a42ed 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -24,33 +24,20 @@ import os import time +import pytest + from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud -# Resource key environment variables follow the 51Degrees convention, which -# is that every one of them starts with "_51DEGREES_RESOURCE_KEY". The name -# used before the convention was adopted is still read, so an existing setup -# keeps working. -RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" -LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - -MISSING_KEY_MESSAGE = ( - "No resource key found, so the tests that call the cloud service " - f"cannot run. Set the environment variable '{RESOURCE_KEY_ENV_VAR}' " - f"(the older name '{LEGACY_RESOURCE_KEY_ENV_VAR}' is still read). " - "Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-tests-test_cloud.py&utm_term=resource-key-required") - +from .live_cloud import find_resource_key, no_resource_key -def get_resource_key(): - """! - The resource key from the environment, or None when neither the aligned - nor the older variable is set. - """ +pytestmark = pytest.mark.live - return (os.environ.get(RESOURCE_KEY_ENV_VAR) - or os.environ.get(LEGACY_RESOURCE_KEY_ENV_VAR)) +NO_KEY = no_resource_key( + "the device product", + "device detection properties such as IsMobile, and their metadata", + several=True) mobile_ua = "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114" @@ -59,10 +46,10 @@ def get_resource_key(): class DeviceDetectionTests(unittest.TestCase): def setUp(self): - self.resource_key = get_resource_key() + self.resource_key = find_resource_key() if not self.resource_key: - self.skipTest(MISSING_KEY_MESSAGE) + self.fail(NO_KEY) def test_pipeline_builder_cloud_engine_init(self): diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py index 43bbeb34..8a525e71 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_properties.py @@ -23,6 +23,8 @@ import os import unittest +import pytest + from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud @@ -32,6 +34,10 @@ is_same_type, ) +from .live_cloud import find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + header_file_path = "./tests/51Degrees.csv" mobile_ua = ("Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) " "AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile" @@ -43,71 +49,54 @@ # TODO remove setheader properties from this list once UACH datafile is released. exclude_properties = ["setheaderbrowseraccept-ch", "setheaderplatformaccept-ch", "setheaderhardwareaccept-ch"] -# Resource key environment variables follow the 51Degrees convention, which -# is that every one of them starts with "_51DEGREES_RESOURCE_KEY". The name -# used before the convention was adopted is still read, so an existing setup -# keeps working. -RESOURCE_KEY_ENV_VAR = "_51DEGREES_RESOURCE_KEY" -LEGACY_RESOURCE_KEY_ENV_VAR = "resource_key" - -resource_key = (os.environ.get(RESOURCE_KEY_ENV_VAR) - or os.environ.get(LEGACY_RESOURCE_KEY_ENV_VAR)) - -if not resource_key: - # Skipping rather than raising, so a run without a key shows as skipped - # and names the variable, instead of turning into a collection error. - import pytest - - pytest.skip( - "No resource key found, so the tests that call the cloud service " - f"cannot run. Set the environment variable '{RESOURCE_KEY_ENV_VAR}' " - f"(the older name '{LEGACY_RESOURCE_KEY_ENV_VAR}' is still read). " - "Create a resource key for free at " - "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud-tests-test_properties.py&utm_term=resource-key-required", - allow_module_level=True) - -if not os.path.isfile(header_file_path): - # The property list comes from an asset the build fetches with a licence - # key. Without it these tests cannot run, so skip rather than turn a - # missing asset into a failure. - import pytest - - pytest.skip( - f"The property list file '{header_file_path}' is not present, so " - "the property coverage tests cannot run. It is fetched by " - "ci/fetch-assets.ps1 when a device detection licence key is " - "available.", - allow_module_level=True) - -# Get Properties list -properties_list = get_properties_from_header_file(header_file_path) - -# Create a simple pipeline to access the engine with and process it with flow data -cloudRequestEngine = CloudRequestEngine({"resource_key": resource_key}) -deviceDetectionCloudEngine = DeviceDetectionCloud() -pipeline = PipelineBuilder() \ - .add(cloudRequestEngine) \ - .add(deviceDetectionCloudEngine) \ - .build() - +NO_KEY = no_resource_key( + "the device product", + "every device detection property listed in " + header_file_path, + several=True) + class PropertyTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.resource_key = find_resource_key() + cls.pipeline = None + if cls.resource_key and os.path.isfile(header_file_path): + cls.properties_list = get_properties_from_header_file( + header_file_path) + cls.engine = DeviceDetectionCloud() + cls.pipeline = PipelineBuilder() \ + .add(CloudRequestEngine({"resource_key": cls.resource_key})) \ + .add(cls.engine) \ + .build() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + if self.pipeline is None: + # The property list comes from an asset the build fetches with + # a licence key, so a run without it says which file is missing. + self.skipTest( + f"The property list file '{header_file_path}' is not " + "present, so the property coverage tests cannot run. It is " + "fetched by ci/fetch-assets.ps1 when a device detection " + "licence key is available.") + def test_available_properties(self): """! Tests whether the all the properties present in the engine when initialised with a resource key are accessible. """ - flowData = pipeline.create_flowdata() + flowData = self.pipeline.create_flowdata() flowData.evidence.add("header.user-agent", mobile_ua) flowData.process() - elementData = flowData.get(deviceDetectionCloudEngine.datakey) + elementData = flowData.get(self.engine.datakey) # Get dictionary of elementdata properties dd_property_dict = elementData.as_dictionary() #RUn test checks on all the properties available in header file - for property in properties_list: + for property in self.properties_list: property = property.lower() if(property not in exclude_properties): if(property in dd_property_dict): @@ -128,16 +117,16 @@ def test_value_types(self): Tests value types of the properties present present in the engine """ - flowData = pipeline.create_flowdata() + flowData = self.pipeline.create_flowdata() flowData.evidence.add("header.user-agent", chrome_ua) flowData.process() - elementData = flowData.get(deviceDetectionCloudEngine.datakey) + elementData = flowData.get(self.engine.datakey) # Get dictionary of elementdata properties dd_property_dict = elementData.as_dictionary() # Get properties from the engine - properties_list = deviceDetectionCloudEngine.get_properties() + properties_list = self.engine.get_properties() # Run test check valuetypes of properties for propertykey, propertymeta in properties_list.items(): diff --git a/device-detection/fiftyone_devicedetection_cloud/tox.ini b/device-detection/fiftyone_devicedetection_cloud/tox.ini index a0d62c6b..c58bf36f 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tox.ini +++ b/device-detection/fiftyone_devicedetection_cloud/tox.ini @@ -21,10 +21,14 @@ deps = -e../../pipeline/fiftyone_pipeline_cloudrequestengine/ commands = pytest --cov={envsitepackagesdir}/fiftyone_devicedetection_cloud {tty:--color=yes} {posargs} +# PYTEST_ADDOPTS carries the marker selection of ../ci/run-unit-tests.ps1, +# which leaves the live tests out of the unit step. pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key + PYTEST_ADDOPTS [testenv:pre-publish] package = skip diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index d25b881e..846ebc97 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -41,7 +41,10 @@ examples as they run from a fresh clone, with no licence key, and check that each one finishes on the free Lite data file and gives the link to ask for the Enterprise data file. -Cloud tests will only run with a valid 51Degrees resource key (see above) set as a resource_key operating system environment variable. +The cloud tests need a valid 51Degrees resource key (see above), and fail +without one. They read it from the environment variable +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`, then +`resource_key`. For example, use following command to set resource_key on Linux: `export resource_key=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py index 79bf78c5..f5e10453 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py @@ -35,6 +35,13 @@ from fiftyone_devicedetection_examples.cloud.gettingstarted_console import GettingStartedConsole from fiftyone_devicedetection_examples.cloud.configurator_console import ConfiguratorConsole +from .live_cloud import find_resource_key, no_resource_key + +NO_KEY = no_resource_key( + "the device product, and the hardware product for the lookups", + "device detection properties, and for the hardware profiles the TAC " + "and native model lookups return", several=True) + # Text that shows an example printed a programming fault rather than a # result. An example that reaches any of these is broken, however little of # the data the resource key is entitled to. @@ -50,17 +57,12 @@ class DeviceDetectionExampleTests(unittest.TestCase): - # Init method - specify a resource key here, or set one in the - # environment variable named by ExampleUtils.RESOURCE_KEY_ENV_VAR. def setUp(self): - self.resource_key = ExampleUtils.get_resource_key() + self.resource_key = find_resource_key() self.logger = Logger() if not self.resource_key: - # Skipping rather than passing, so a run without a key is not - # mistaken for a run that proved something. The message names - # the variable that was wanted. - self.skipTest(ExampleUtils.get_missing_resource_key_message()) + self.fail(NO_KEY) def run_example(self, example): """! diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 0b6222c0..b0932b5a 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -33,6 +33,7 @@ commands = {[common]commands} pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key run_performance_tests diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index 5e50e871..f57e296c 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -14,17 +14,18 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so cloud integration tests will not run." -} -else { - # Says what that key is entitled to, so a run with no IP data is read - # as an entitlement gap rather than as broken code. Prints product - # names and tiers only, never the key. Found through this script's own - # folder, because common-ci runs it from the common-ci checkout, where - # './ci/report-key-products.ps1' does not exist. - & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } +# Says what that key is entitled to, so a run with no IP data is read as an +# entitlement gap rather than as broken code. Prints product names and tiers +# only, never the key. Found through this script's own folder, because +# common-ci runs it from the common-ci checkout, where +# './ci/report-key-products.ps1' does not exist. +& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey + # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get # installed. That's why we install it here, as a special case. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index 0ada1f94..df742cdb 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -29,6 +29,8 @@ from fiftyone_ipintelligence_shared.example_constants import ASN_DATAFILE_NAME from fiftyone_ipintelligence_shared.example_constants import EVIDENCE_FILE_NAME +from .live_cloud import find_resource_key, no_resource_key + # The tests run against the freely available ASN data file the data # repository links to, so they only rely on properties that the free data # file provides (location properties such as Latitude, Longitude and Areas @@ -36,7 +38,14 @@ data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) evidence_file = ExampleUtils.find_file(EVIDENCE_FILE_NAME) truth_file = ExampleUtils.find_file("geoip_comparison_evidence.csv") -resource_key = ExampleUtils.get_resource_key() +resource_key = find_resource_key() + +NO_IP_KEY = no_resource_key( + "the ip product", + "IP Intelligence properties such as RegisteredName, and their metadata") +NO_MIXED_KEY = no_resource_key( + "the device and ip products", + "device detection and IP Intelligence properties together") # The mixed examples combine IP intelligence with device detection, which is # an optional dependency. Their tests only run when it is installed. @@ -146,23 +155,23 @@ def test_onpremise_compare_console(self): # header + up to 20 records self.assertGreater(len(output.getvalue().splitlines()), 1) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_gettingstarted_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.gettingstarted_console import GettingStartedConsole example = GettingStartedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_metadata_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.metadata_console import MetadataConsole example = MetadataConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) - @unittest.skipUnless( - bool(resource_key), "No resource_key environment variable set") def test_cloud_getallproperties_console(self): + if not resource_key: + self.fail(NO_IP_KEY) from fiftyone_ipintelligence_examples.cloud.getallproperties_console import GetAllPropertiesConsole example = GetAllPropertiesConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) @@ -176,9 +185,10 @@ def test_onpremise_mixed_console(self): example.run(hash_file, data_file, Logger(min_level="error"), lambda x: None) @unittest.skipUnless( - device_detection_available and bool(resource_key), - "device detection package or resource_key not available") + device_detection_available, "device detection package not available") def test_cloud_mixed_console(self): + if not resource_key: + self.fail(NO_MIXED_KEY) from fiftyone_ipintelligence_examples.cloud.mixed_console import MixedConsole example = MixedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index 3bc4fbd3..459f1a48 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -37,6 +37,8 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_ipintelligence_examples {tty:--color=yes} {posargs} pass_env = + _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE resource_key license_key IPINTELLIGENCEDATAFILE diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 1af0c1c0..bbbdb0bb 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -6,8 +6,9 @@ param ( ) if (!$Keys.TestResourceKey) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No Resource Key::No resource key was provided, so integration tests will not run." - return + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index 0ee466aa..5539b7ea 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,6 +5,12 @@ param ( [Hashtable]$Keys ) +if (!$Keys.TestResourceKey) { + throw "No TestResourceKey was passed, so the cloud tests cannot call " + + "the cloud service. The workflows set it from the " + + "_51DEGREES_RESOURCE_KEY_BESPOKE secret." +} + # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -16,9 +22,23 @@ param ( # not a licence key for the cloud service, so the examples' licence-key tests # failed with it. They are not given it, and skip themselves as they did when # pipeline was its own repository, until a cloud licence key is passed. -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" +# +# The 51Did package runs only its tests marked live here, as its others ran +# in the unit step. Its tier tests need a key for another tier, which only +# the Cloud 51Did workflow has. Only a package whose tox.ini passes +# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which +# build-project.ps1 copies, and the publish workflow's test job does not run +# that script, so it is copied here too. +& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "live and not tier"' +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/pipeline/ci/run-unit-tests.ps1 b/pipeline/ci/run-unit-tests.ps1 index c5af008e..441a2bcc 100644 --- a/pipeline/ci/run-unit-tests.ps1 +++ b/pipeline/ci/run-unit-tests.ps1 @@ -3,7 +3,16 @@ param ( [string]$RepoName ) -$packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" -./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +# The tests marked live call the cloud service, and no key reaches the unit +# step, so they are left to run-integration-tests.ps1. Only a package whose +# tox.ini passes PYTEST_ADDOPTS sees the selection. +$addopts = $env:PYTEST_ADDOPTS +$env:PYTEST_ADDOPTS = '-m "not live"' +try { + $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" + ./python/run-unit-tests.ps1 -RepoName $RepoName -Packages $packages +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index cd8b5b28..58b3ec35 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -90,8 +90,10 @@ then open . Set `PORT` to listen on another port. .venv/bin/python -m pytest ``` -The tests that call the cloud service skip themselves when no resource key -is set. +The tests that call the cloud service with a resource key fail when none is +set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY`, +`_51DEGREES_RESOURCE_KEY_BESPOKE` and `resource_key`, in that order. The +tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py index 74d791c1..60ec92d8 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py @@ -29,18 +29,21 @@ from fiftyone_pipeline_examples.example_utils import ExampleUtils +from live_cloud import find_resource_key, no_resource_key + +NO_KEY = no_resource_key( + "the device, ip and countrynamestranslated products", + "device detection and IP Intelligence properties, and the translated " + "country names", several=True) + # The ways a test can reach the cloud service, as the keyword arguments the -# examples' create_pipeline takes. Each is skipped when its key is not set. +# examples' create_pipeline takes, each passed through credentials(). A test +# with a resource key fails when none is set. A test with a licence key is +# skipped when none is set, because CI has no cloud licence key to give it. KEYS = [ pytest.param( - {"resource_key": ExampleUtils.get_resource_key()}, - id="resource key", - marks=pytest.mark.skipif( - not ExampleUtils.get_resource_key(), - reason=f"Set {ExampleUtils.RESOURCE_KEY_ENV_VAR} to a resource " - "key carrying device detection and IP intelligence " - "properties to run the cloud tests with a resource " - "key.")), + {"resource_key": find_resource_key()}, + id="resource key"), pytest.param( {"license_key": ExampleUtils.get_license_key()}, id="licence key", @@ -53,6 +56,15 @@ ] +def credentials(keys): + """The keyword arguments from KEYS, failing the test when they hold a + resource key that is not set.""" + + if "resource_key" in keys and not keys["resource_key"]: + pytest.fail(NO_KEY, pytrace=False) + return keys + + def cell_after(page, label): """ The text a browser shows in the table cell after the one holding label, diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py index 190d940e..f4cb7bc4 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_console.py @@ -34,7 +34,7 @@ GettingStartedConsole, create_pipeline) from fiftyone_pipeline_examples.example_utils import ExampleUtils -from helpers import KEYS +from helpers import KEYS, credentials SEPARATOR = "=" * 79 @@ -45,7 +45,7 @@ def output(request): pipeline = create_pipeline( Logger(min_level="info"), cloud_endpoint=ExampleUtils.get_cloud_endpoint(), - **request.param) + **credentials(request.param)) GettingStartedConsole().run(pipeline, lines.append) return "\n".join(lines) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py index c3f7eb6d..fa66520c 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/test_cloud_mixed_web.py @@ -37,7 +37,7 @@ create_app, create_pipeline) from fiftyone_pipeline_examples.example_utils import ExampleUtils -from helpers import KEYS, cell_after, country_options +from helpers import KEYS, cell_after, country_options, credentials DESKTOP_CHROME = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " @@ -51,7 +51,7 @@ @pytest.fixture(scope="module", params=KEYS) def key(request): - return request.param + return credentials(request.param) @pytest.fixture(scope="module") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index 1fae6a61..56e9bf52 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -29,15 +29,16 @@ deps = commands = pytest --cov={envsitepackagesdir}/fiftyone_pipeline_examples {tty:--color=yes} {posargs} # tox runs the tests in a clean environment and strips anything it is not -# told to keep, so without this the cloud tests never see a key and skip -# themselves on every run, which reads as a pass. resource_key and -# license_key are the names common-ci sets, from the secrets named in -# ci/run-integration-tests.ps1. The _51DEGREES names are the ones the -# examples document and the Examples workflow sets. The last three name a -# cloud service other than the public one: the first two are read by the -# examples, the third by the cloud request engine underneath them. +# told to keep, so without this the cloud tests never see a key and fail on +# every run. resource_key and license_key are the names common-ci sets, from +# the secrets named in ci/run-integration-tests.ps1. The _51DEGREES names are +# the ones the examples document, the Examples workflow sets and +# tests/live_cloud.py reads. The last three name a cloud service other than the +# public one: the first two are read by the examples, the third by the cloud +# request engine underneath them. pass_env = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE _51DEGREES_LICENSE_KEY resource_key license_key diff --git a/pipeline/fiftyone_pipeline_did/tests/conftest.py b/pipeline/fiftyone_pipeline_did/tests/conftest.py new file mode 100644 index 00000000..8dd5e4f9 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/conftest.py @@ -0,0 +1,38 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The markers that keep the live cloud tests out of the unit test run. +# pipeline/ci/run-unit-tests.ps1 deselects "live", and +# pipeline/ci/run-integration-tests.ps1 runs "live and not tier", because no +# key for another tier reaches it. The Cloud 51Did workflow runs the tier +# tests with the Basic key. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + config.addinivalue_line( + "markers", + "tier: asserts what a key on one tier is entitled to, and needs the " + "key for that tier") diff --git a/pipeline/fiftyone_pipeline_did/tests/live_cloud.py b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py new file mode 100644 index 00000000..239a8852 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/live_cloud.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Where the tests that call the live 51Degrees cloud service find their +resource key, and what they fail with when there is none or when the +answer lacks what they asked for. + +The same file is in each test folder whose tests call the live cloud +service. There is no fallback to a skip, because a cloud test that passes +without calling the cloud has proved nothing. +""" + +import os + +# Tried in this order. _51DEGREES_RESOURCE_KEY is first because the Cloud +# 51Did workflow sets it to each key in turn, and resource_key last because +# it is the older name, which common-ci still sets. +RESOURCE_KEY_NAMES = ( + "_51DEGREES_RESOURCE_KEY", + "_51DEGREES_RESOURCE_KEY_BESPOKE", + "resource_key", +) + +ACCESSIBLE_PROPERTIES_URL = ( + "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") + +CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" + + +def find_resource_key(names=RESOURCE_KEY_NAMES): + """The first resource key set under any of the names, or None.""" + for name in names: + value = os.environ.get(name, "").strip() + if value: + return value + return None + + +def no_resource_key(product, asks_for, several=False, + names=RESOURCE_KEY_NAMES): + """What to fail with when no resource key is set. + + product names the product the key has to carry, with its article, as in + "the device product". asks_for says what the test asks the service for. + several is true where the message stands for a group of tests. + """ + if several: + opening = ("These tests call the live 51Degrees cloud service, so " + "they need a resource key") + asker = "the tests ask" + else: + opening = ("This test calls the live 51Degrees cloud service, so " + "it needs a resource key") + asker = "the test asks" + return ( + f"{opening} and there is nothing to fall back on. None of " + f"{', '.join(names)} is set, tried in that order. The key has to " + f"carry {product}, because {asker} the service for {asks_for}. To " + f"see the products a key carries, open {ACCESSIBLE_PROPERTIES_URL}. " + f"Create a key at {CREATE_KEY_URL}.") + + +def answered_without(thing, reason, product): + """What to fail with when the cloud answered without the thing asked + for. reason is the cloud's null reason, or None when it gave none.""" + return ( + f"The cloud answered without {thing} " + f"({reason or 'no reason given'}), which is what a key that does " + f"not carry {product} gets.") diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py index 23d8fe90..6a41e265 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_client_live.py @@ -20,25 +20,27 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -"""Live tests against the cloud, run only when a resource key is set. - -The key is read from ``resource_key``, the variable the repository's other -live tests and its CI use, or from ``_51DEGREES_RESOURCE_KEY`` as the -example reads it. Without either the tests are skipped, not failed. An -optional licence key is read from ``license_key`` or -``_51DEGREES_LICENSE_KEY``, and the endpoint from ``FOD_CLOUD_API_URL`` as -everywhere else. Every test here costs uses against the resource key. +"""Live tests against the 51Degrees cloud service. + +The resource key is read from the names in ``live_cloud.py``, and every +test fails when none is set. The key has to carry the FODid product, and +the CloudV5FODiD product for the marketing usages, which +``test_did_tier_live.py`` shows a Basic key does not get. An optional +licence key is read from ``license_key`` or ``_51DEGREES_LICENSE_KEY``, and +the endpoint from ``FOD_CLOUD_API_URL`` as everywhere else. Every test here +costs uses against the resource key. """ import asyncio import json import os -import sys import unittest import urllib.error import urllib.parse import urllib.request +import pytest + from fiftyone_pipeline_did import ( ContextResult, DidClient, @@ -49,11 +51,22 @@ ) from fiftyone_pipeline_did.did_client import USER_AGENT -RESOURCE_KEY = os.environ.get("resource_key") \ - or os.environ.get("_51DEGREES_RESOURCE_KEY") +from .live_cloud import answered_without, find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + +RESOURCE_KEY = find_resource_key() LICENCE_KEY = os.environ.get("license_key") \ or os.environ.get("_51DEGREES_LICENSE_KEY") +FODID = "the FODid product" +MARKETING = "the CloudV5FODiD product" + +NO_KEY = no_resource_key( + FODID + ", and " + MARKETING + " for the marketing usages", + "FODiD.IdProbGlobal and FODiD.IdProbLic under each id.usage", + several=True) + def run(coroutine): """Runs one client call to completion on a fresh event loop, as the @@ -62,14 +75,27 @@ def run(coroutine): return asyncio.run(coroutine) -@unittest.skipUnless( - RESOURCE_KEY, - "set resource_key (or _51DEGREES_RESOURCE_KEY) to run the live tests") +def null_reason(fodid): + """The reason the cloud gave for answering without a 51Did.""" + return fodid.get("idprobglobalnullreason") \ + or fodid.get("idproblicnullreason") + + +def no_creator_context(endpoint, detail): + """What to fail with when the host does not offer the creator context, + which the 51Degrees cloud does.""" + return ("The cloud service at {0} {1}, so the creator context could not " + "be checked. Point FOD_CLOUD_API_URL at a service that offers " + "it, or leave it unset for the 51Degrees cloud.").format( + endpoint, detail) + + class DidClientLiveTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.client = DidClient(RESOURCE_KEY, LICENCE_KEY) + def setUp(self): + if not RESOURCE_KEY: + self.fail(NO_KEY) + self.client = DidClient(RESOURCE_KEY, LICENCE_KEY) def create(self): """Creates a 51Did through the cloud ``json`` endpoint, the route @@ -88,10 +114,10 @@ def create(self): body = json.loads(response.read().decode("utf-8")) fodid = body.get("fodid") or {} value = fodid.get("idproblic") or fodid.get("idprobglobal") - self.assertTrue( - value, "the resource key must include a 51Did property " - "(idprobglobal or idproblic); the json answer was: " - + json.dumps(body)[:300]) + if not value: + self.fail(answered_without( + "a 51Did for id.usage=non-marketing", null_reason(fodid), + FODID)) return FodId.from_base64(value) def test_created_identifier_verifies_offline_and_through_the_cloud(self): @@ -108,8 +134,10 @@ def test_garbage_result_redeems_as_unreadable(self): try: result = run(self.client.redeem(fod_id, "not-base64url!!", "x")) except DidNotSupportedError as error: - self.skipTest("the host does not offer the creator context: " - + str(error)) + self.fail(no_creator_context( + self.client.endpoint, + "answered that it does not offer the creator context (" + + str(error) + ")")) self.assertEqual(200, result.status_code) self.assertEqual(ContextResult.UNREADABLE, result.context) @@ -123,9 +151,8 @@ def test_garbage_result_redeems_as_unreadable(self): def identifiers_for(self, name, value): """Asks the ``json`` endpoint for a 51Did with the given query - parameter and returns every identifier it answered with. An empty - list means the resource key is not entitled to that usage, which - the caller reports rather than fails.""" + parameter, and returns every identifier it answered with and the + ``fodid`` member, which holds the reason when there is none.""" url = "{0}{1}.json?{2}&values=FODiD.IdProbGlobal" \ "&values=FODiD.IdProbLic".format( self.client.endpoint, @@ -137,11 +164,12 @@ def identifiers_for(self, name, value): with urllib.request.urlopen(request, timeout=30) as response: body = json.loads(response.read().decode("utf-8")) fodid = body.get("fodid") or {} - return [ + identifiers = [ FodId.from_base64(fodid[field]) for field in ("idprobglobal", "idproblic") if fodid.get(field) ] + return identifiers, fodid def assert_aligned(self, label, fod_id, usage, terms, indirect): """Asserts the terms and every field the flags byte carries, read @@ -170,35 +198,23 @@ def test_every_usage_reads_back_the_terms_and_flags(self): ends at the match key and reads as no terms. """ cases = [ - ("non-marketing", Usage.NON_MARKETING, None), - ("standard", Usage.STANDARD, self.MODEL_TERMS_FOR_MARKETING_2), + ("non-marketing", Usage.NON_MARKETING, None, FODID), + ("standard", Usage.STANDARD, self.MODEL_TERMS_FOR_MARKETING_2, + MARKETING), ("personalized", Usage.PERSONALIZED, - self.MODEL_TERMS_FOR_MARKETING_2), + self.MODEL_TERMS_FOR_MARKETING_2, MARKETING), ] - checked = 0 - for name, usage, terms in cases: - identifiers = self.identifiers_for("id.usage", name) + for name, usage, terms, product in cases: + identifiers, fodid = self.identifiers_for("id.usage", name) if not identifiers: - print("id.usage={0}: no identifier returned, so this key is " - "not entitled to that usage.".format(name), - file=sys.stderr) - continue + self.fail(answered_without( + "a 51Did for id.usage=" + name, null_reason(fodid), + product)) for index, fod_id in enumerate(identifiers): self.assert_aligned( "{0}[{1}]".format(name, index), fod_id, usage, terms, False) - if terms is not None: - checked += len(identifiers) - - # Reported as a skip rather than printed, because a runner shows - # the counts and not the output, so a run that proved nothing would - # otherwise look identical to one that proved everything. - if checked == 0: - self.skipTest( - "this resource key returned no marketing 51Did, so no terms " - "address was read and this run did not prove it; use a key " - "entitled to the standard or personalized usage") def test_consent_string_sets_the_usage_is_indirect_bit(self): """A consent management platform sends an IAB TCF consent string @@ -219,39 +235,28 @@ def test_consent_string_sets_the_usage_is_indirect_bit(self): the Appendix 1 standard set of 1, 2, 7, 8 and 11. """ cases = [ - ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage.PERSONALIZED), - ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage.STANDARD), + ("AAAAAAAAAAAAAAAAAAAAAAAAAP_w", Usage.PERSONALIZED, + "personalized"), + ("AAAAAAAAAAAAAAAAAAAAAAAAAMMg", Usage.STANDARD, "standard"), ] - proven = 0 - for tc_string, usage in cases: + for tc_string, usage, name in cases: # No id.usage is sent. A stated usage wins over a consent # string, so sending one would leave the bit clear and this # would prove the opposite of what it says. - identifiers = self.identifiers_for("tcstring", tc_string) + identifiers, fodid = self.identifiers_for("tcstring", tc_string) if not identifiers: - print("consent string granting {0}: no identifier returned, " - "so this key is not entitled to that marketing " - "usage.".format(usage), file=sys.stderr) - continue + self.fail(answered_without( + "a 51Did for a consent string granting the " + name + + " usage", null_reason(fodid), MARKETING)) for index, fod_id in enumerate(identifiers): # A consent string granting a marketing usage produces a # marketing identifier, so the terms travel with it too. self.assert_aligned( - "consent/{0}[{1}]".format(usage, index), fod_id, usage, + "consent/{0}[{1}]".format(name, index), fod_id, usage, self.MODEL_TERMS_FOR_MARKETING_2, True) - proven += len(identifiers) - - # Same reasoning as the usage test above. - if proven == 0: - self.skipTest( - "this resource key returned no identifier for either consent " - "string, so the usage is indirect bit was never read and " - "this run did not prove it") - -@unittest.skipUnless( - RESOURCE_KEY, - "set resource_key (or _51DEGREES_RESOURCE_KEY) to run the live tests") + + class BrowserCreatorContextLiveTests(unittest.TestCase): """The whole browser path against the live service, as a page runs it. @@ -264,6 +269,10 @@ class BrowserCreatorContextLiveTests(unittest.TestCase): service's contract with every one of these packages, and a name the service adds, drops or renames has to be seen here rather than by a customer. + + Without a licence key the sealed result cannot be opened, so each test + then checks that it redeems as unreadable, which is what a caller with + no licence key gets. """ # A browser the service will judge as a browser, and a second one @@ -297,9 +306,10 @@ class BrowserCreatorContextLiveTests(unittest.TestCase): "transport", "device", "browserip", "connectionip", "asn", "platformname", "platformversion", "browsername", "browserversion") - @classmethod - def setUpClass(cls): - cls.client = DidClient(RESOURCE_KEY, LICENCE_KEY) + def setUp(self): + if not RESOURCE_KEY: + self.fail(NO_KEY) + self.client = DidClient(RESOURCE_KEY, LICENCE_KEY) def get(self, url, user_agent): request = urllib.request.Request( @@ -325,14 +335,10 @@ def create(self, usage="standard"): value = fodid.get("idproblic") or fodid.get("idprobglobal") if not value: # The probabilistic reason in full, because it names every - # snippet result the service is still waiting for, and a cut - # dump of the whole member hides them behind the email ones. - reason = fodid.get("idprobglobalnullreason") \ - or fodid.get("idproblicnullreason") \ - or json.dumps(fodid)[:300] - self.skipTest( - "this resource key returned no identifier for a browser " - "shaped request: " + reason) + # snippet result the service is still waiting for. + self.fail(answered_without( + "a 51Did for a browser shaped request with id.usage=" + + usage, null_reason(fodid), MARKETING)) return FodId.from_base64(value) def verify_context(self, fod_id, user_agent): @@ -346,13 +352,16 @@ def verify_context(self, fod_id, user_agent): urllib.parse.urlencode(query)) status, body = self.get(url, user_agent) if status == 404: - self.skipTest("the host does not offer the creator context") + self.fail(no_creator_context( + self.client.endpoint, + "answered 404 from id/verify-context")) self.assertEqual(200, status, body[:300]) answer = json.loads(body) if "result" not in answer: - self.skipTest( - "this host holds no context secret, so it answered in the " - "open with " + body[:200]) + self.fail(no_creator_context( + self.client.endpoint, + "holds no creator context secret, so it answered in the " + "open with " + body[:200])) return answer["result"] def test_a_page_creates_reads_back_verifies_and_redeems(self): @@ -364,19 +373,18 @@ def test_a_page_creates_reads_back_verifies_and_redeems(self): result = self.verify_context(fod_id, self.CHROME) redeemed = run(self.client.redeem(fod_id, result)) if not self.client.has_licence_key: - # Without a licence key the sealed result cannot be opened, - # which the service reports as unreadable. self.assertEqual(ContextResult.UNREADABLE, redeemed.context) - self.skipTest("set license_key to redeem a sealed result") + return self.assertEqual(ContextResult.VERIFIED, redeemed.context) self.assertEqual("verified", redeemed.signature.value) def test_another_browser_redeems_as_a_mismatch_naming_every_factor(self): - if not LICENCE_KEY: - self.skipTest("set license_key to redeem a sealed result") fod_id = self.create() result = self.verify_context(fod_id, self.SAFARI) redeemed = run(self.client.redeem(fod_id, result)) + if not self.client.has_licence_key: + self.assertEqual(ContextResult.UNREADABLE, redeemed.context) + return self.assertEqual(ContextResult.MISMATCH, redeemed.context) self.assertEqual("verified", redeemed.signature.value, "the identifier is genuine whichever browser " diff --git a/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py b/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py new file mode 100644 index 00000000..e52b5f98 --- /dev/null +++ b/pipeline/fiftyone_pipeline_did/tests/test_did_tier_live.py @@ -0,0 +1,113 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""What a key on the Basic tier is entitled to, asked of the live cloud. + +A Basic key carries the FODid product, so it gets a 51Did for the +non-marketing usage. It does not carry the CloudV5FODiD product, so the +service refuses it the standard and personalized usages and says which +product is missing. The key is read from ``_51DEGREES_RESOURCE_KEY_BASIC`` +only, because these tests assert what that one tier gets, and they fail +when it is not set. +""" + +import json +import urllib.parse +import urllib.request +import unittest + +import pytest + +from fiftyone_pipeline_did import DidClient, FodId, Usage +from fiftyone_pipeline_did.did_client import USER_AGENT + +from .live_cloud import answered_without, find_resource_key, no_resource_key + +pytestmark = [pytest.mark.live, pytest.mark.tier] + +BASIC_KEY_NAMES = ("_51DEGREES_RESOURCE_KEY_BASIC",) +BASIC_KEY = find_resource_key(BASIC_KEY_NAMES) + +FIELDS = ("idprobglobal", "idproblic") + + +class BasicTierLiveTests(unittest.TestCase): + + def setUp(self): + if not BASIC_KEY: + self.fail(no_resource_key( + "the FODid product on the Basic tier", + "a 51Did under each id.usage, expecting the marketing " + "usages to be refused", several=True, + names=BASIC_KEY_NAMES)) + self.endpoint = DidClient(BASIC_KEY).endpoint + + def fodid_for(self, usage): + """The ``fodid`` member of the ``json`` endpoint's answer for the + usage.""" + url = "{0}{1}.json?{2}".format( + self.endpoint, urllib.parse.quote(BASIC_KEY, safe=""), + urllib.parse.urlencode([ + ("id.usage", usage), + ("values", "FODiD.IdProbGlobal"), + ("values", "FODiD.IdProbLic")])) + request = urllib.request.Request( + url, data=b"", headers={"User-Agent": USER_AGENT}, + method="POST") + with urllib.request.urlopen(request, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + return body.get("fodid") or {} + + def test_non_marketing_usage_gets_a_51did(self): + fodid = self.fodid_for("non-marketing") + for field in FIELDS: + if not fodid.get(field): + self.fail(answered_without( + field + " for id.usage=non-marketing", + fodid.get(field + "nullreason"), "the FODid product")) + self.assertEqual( + Usage.NON_MARKETING, FodId.from_base64(fodid[field]).usage, + field) + + def test_marketing_usages_are_refused_naming_the_product(self): + for usage in ("standard", "personalized"): + fodid = self.fodid_for(usage) + for field in FIELDS: + self.assertFalse( + fodid.get(field), + "A Basic key got {0} for id.usage={1}, which only a key " + "that carries the CloudV5FODiD product should get. Check " + "the key is still on the Basic tier.".format( + field, usage)) + name = "IdProbGlobal" if field == "idprobglobal" \ + else "IdProbLic" + self.assertEqual( + "Can not form {0}. For marketing (personalized or " + "standard) usages a CloudV5FODiD product needs to be a " + "part of the license.".format(name), + fodid.get(field + "nullreason"), + "the reason given for {0} with id.usage={1}".format( + field, usage)) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_did/tox.ini b/pipeline/fiftyone_pipeline_did/tox.ini index d80d881b..edf34540 100644 --- a/pipeline/fiftyone_pipeline_did/tox.ini +++ b/pipeline/fiftyone_pipeline_did/tox.ini @@ -12,13 +12,17 @@ package = wheel wheel_build_env = .pkg # tox runs the tests in a clean environment and strips anything it is not # told to keep, so without this the live cloud tests never see the resource -# key and skip themselves on every run, which reads as a pass. +# key and fail on every run. PYTEST_ADDOPTS carries the markers the CI +# scripts select with. passenv = _51DEGREES_RESOURCE_KEY + _51DEGREES_RESOURCE_KEY_BESPOKE + _51DEGREES_RESOURCE_KEY_BASIC _51DEGREES_LICENSE_KEY resource_key license_key FOD_CLOUD_API_URL + PYTEST_ADDOPTS deps = {[common]deps} # owid is provided by the owid-python submodule (../owid-python), not From bb11a56329a38b060b1785ff1b18e4112fd8043f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 06:16:24 +0000 Subject: [PATCH 20/26] Make the example profile comments and the Lite file name match the code (#65) - The IP Intelligence data file update example now names the LowMemory profile for the pipeline that downloads and reloads the Enterprise data file, rather than taking the engine's default. - The Device Detection getting started example's comment said it uses the low memory profile, but the code sets MaxPerformance. The comment now says MaxPerformance, and why it suits the free Lite data file. - The IP Intelligence performance example's header said worker processes share a memory mapped data file under LowMemory, as if that were the default. It now says the default MaxPerformance loads the whole file into each worker, and that -p LowMemory lets the workers share the operating system's file cache. - The IP Intelligence package readme named the Lite data file 51Degrees-LiteIpiV41.ipi, which is CI's name for it. The data repository's scripts save it as 51Degrees-LiteV41.ipi. --- .../onpremise/gettingstarted_console.py | 8 +++++--- ip-intelligence/fiftyone_ipintelligence/readme.md | 2 +- .../onpremise/datafileupdate_console.py | 3 +++ .../onpremise/performance_console.py | 9 ++++++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py index 12c4cff6..4742eed1 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py @@ -65,9 +65,11 @@ def run(self, data_file, logger, output): # https://51degrees.com/documentation/_pipeline_api__concepts__configuration__builders__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run pipeline = DeviceDetectionPipelineBuilder( data_file_path = data_file, - # We use the low memory profile as its performance is - # sufficient for this example. See the documentation for - # more detail on this and other configuration options: + # We use the MaxPerformance profile, which loads the whole data + # file into memory. The free Lite data file is about 35 MB, so + # that costs little memory and gives the fastest detections. + # See the documentation for more detail on this and other + # configuration options: # https://51degrees.com/documentation/_device_detection__features__performance_options.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run # https://51degrees.com/documentation/_pipeline_api__features__automatic_datafile_updates.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run # https://51degrees.com/documentation/_pipeline_api__features__usage_sharing.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=run diff --git a/ip-intelligence/fiftyone_ipintelligence/readme.md b/ip-intelligence/fiftyone_ipintelligence/readme.md index 1b1c9eec..4b546d11 100644 --- a/ip-intelligence/fiftyone_ipintelligence/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence/readme.md @@ -25,7 +25,7 @@ pipeline = IpIntelligencePipelineBuilder(resource_key="YOUR_RESOURCE_KEY").build # On-premise pipeline = IpIntelligencePipelineBuilder( - data_file_path="51Degrees-LiteIpiV41.ipi", + data_file_path="51Degrees-LiteV41.ipi", licence_keys="", usage_sharing=False).build() diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py index a11c17a3..54971693 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/datafileupdate_console.py @@ -188,6 +188,9 @@ def run(self, data_file, license_key, interactive, logger, output): # is configured to use a temporary copy. data_file_path = data_file, create_temp_data_copy = True, + # The Enterprise data file is large, so the LowMemory profile + # reads it from disk rather than holding parts of it in memory. + performance_profile = "LowMemory", # The update listener notifies us when an update is complete. data_file_update_service = update_service, # A license key is required for automatic updates. diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py index c76f930a..e1001011 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py @@ -35,9 +35,12 @@ # # The native engine holds the Python GIL for the duration of each lookup, so # threads cannot run lookups in parallel. Parallelism therefore uses worker -# processes, each building its own pipeline. With the LowMemory profile the -# memory mapped data file is shared between the processes by the operating -# system, so a large enterprise data file is not duplicated per worker. +# processes, each building its own pipeline. With the default MaxPerformance +# profile each worker loads the whole data file into its own memory, so the +# memory used grows with the number of workers. With -p LowMemory each worker +# reads the data file from disk as it needs it, and the operating system's +# file cache holds one copy for all the workers, so a large Enterprise data +# file is not held once per worker. # # Required PyPi Dependencies: # - fiftyone_ipintelligence From f3f0d561610be22467c2fb661531a0d2f4306620 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 06:16:42 +0000 Subject: [PATCH 21/26] Use the Balanced performance profile by default for IP Intelligence (#62) The IP Intelligence on-premise specification gives Balanced as the default performance profile, and the C library's default IP Intelligence configuration is the Balanced one, but the Python engine defaulted to LowMemory. The engine now defaults to "Balanced", which its profile switch already maps to the native Balanced configuration, and the docstrings in the engine and both builders say so. Balanced loads some collections into memory and caches others, where LowMemory reads everything from the data file, so a caller relying on the default uses more memory. Naming "LowMemory" keeps the previous behaviour. A new test, on the free ASN data file, compares the collection settings of the configuration an engine built at its defaults gives the native code with those of each profile, because the engine keeps no name for the profile in use. --- .../ipintelligence_pipelinebuilder.py | 4 +- .../ipintelligence_onpremise.py | 6 +- ...pintelligence_onpremise_pipelinebuilder.py | 4 +- .../tests/test_performance_profile_default.py | 118 ++++++++++++++++++ 4 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py diff --git a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py index 8576e4cf..e17ee5c5 100644 --- a/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence/src/fiftyone_ipintelligence/ipintelligence_pipelinebuilder.py @@ -82,7 +82,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py index 37b45904..9f15784e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py @@ -57,7 +57,7 @@ def __init__( polling_interval=30, update_time_maximum_randomisation=10, create_temp_data_copy=True, - performance_profile="LowMemory", + performance_profile="Balanced", reuse_temp_file=False, concurrency=multiprocessing.cpu_count(), update_on_start=False, @@ -84,7 +84,9 @@ def __init__( @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py index adb27093..182d39a8 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise_pipelinebuilder.py @@ -62,7 +62,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py new file mode 100644 index 00000000..65211ba7 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_performance_profile_default.py @@ -0,0 +1,118 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule import ConfigIpiSwig +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise import IpIntelligenceOnPremise +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise_pipelinebuilder import IpIntelligenceOnPremisePipelineBuilder + +from .constants import TestConstants + +# The free ASN data file, which needs no licence key. +DATA_FILE = os.path.normpath(TestConstants.get_asn_data_file_path()) + +# The collections whose settings the configuration can be asked for. The +# getters for components, properties, values and profiles hand Python a +# setting the configuration owns, which Python then frees, corrupting the +# heap, so those four are left out. +COLLECTIONS = ( + "Strings", "Maps", "Graphs", "ProfileGroups", "ProfileOffsets", + "PropertyTypes", "Graph") + +# Each profile name the engine accepts and the native setter it maps to. +PROFILES = { + "LowMemory": "setLowMemory", + "MaxPerformance": "setMaxPerformance", + "Balanced": "setBalanced", + "BalancedTemp": "setBalancedTemp", + "HighPerformance": "setHighPerformance", +} + + +def collection_settings(config): + """ + Whether each collection is loaded into memory and how many items it + caches, which are what a performance profile sets. + """ + settings = {} + for name in COLLECTIONS: + collection = getattr(config, "get" + name)() + settings[name] = (collection.getLoaded(), collection.getCapacity()) + return settings + + +def profile_settings(setter): + """The collection settings the native code gives a profile.""" + config = ConfigIpiSwig() + getattr(config, setter)() + return collection_settings(config) + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free ASN data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class PerformanceProfileDefaultTests(unittest.TestCase): + """ + The engine keeps no name for the profile in use, so these tests compare + the collection settings of the configuration the engine gives the + native code with the settings each profile has there. MaxPerformance + and LowMemory have the same collection settings, and Balanced and + BalancedTemp do too, so neither pair can be told apart this way. + """ + + def _engine_settings(self, **settings): + engine = IpIntelligenceOnPremise( + data_file_path=DATA_FILE, licence_keys="", **settings) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + self.assertIsNotNone(engine.engine) + return collection_settings(engine.config) + + def test_balanced_and_low_memory_differ(self): + self.assertNotEqual( + profile_settings("setBalanced"), profile_settings("setLowMemory")) + + def test_engine_defaults_use_balanced(self): + self.assertEqual( + profile_settings("setBalanced"), self._engine_settings()) + + def test_builder_defaults_use_balanced(self): + pipeline = IpIntelligenceOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, + licence_keys="").build() + self.addCleanup(pipeline.close) + engine = pipeline.get_element("ip") + self.assertEqual( + profile_settings("setBalanced"), + collection_settings(engine.config)) + + def test_each_named_profile_is_used(self): + for name, setter in PROFILES.items(): + with self.subTest(name): + self.assertEqual( + profile_settings(setter), + self._engine_settings(performance_profile=name)) From d6ebf5ef3f5234ab49f317a07bacd5de04bc4b14 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 12:44:21 +0000 Subject: [PATCH 22/26] Accept the specification's translation option names, and the behavior as text (#64) * Accept the missing translation behavior as text, and refuse an unknown one TranslationEngineBase and Translator stored the behavior they were given without checking it, and compared it with the MissingTranslationBehavior members. A configuration file can only give text, so a configuration naming "EmptyString" or "flow_error" matched no member and a missing translation kept its original value, with nothing to say the setting had been ignored. MissingTranslationBehavior.from_value now returns a member as it is, and turns text naming a member into that member. It takes the specification's names (Original, EmptyString, FlowError) and the members' own names and values, ignoring case and underscores, and raises ValueError for anything else. The engine and the translator use it, so an unknown value stops the engine being built. The parameter is still named behavior. * Accept the specification's name for the missing translation behavior The specification names the Translation Engine's options SourceElementDataKey and MissingTranslationBehavior. Python already took source_element_data_key, but named the second option behavior. TranslationEngineBase and Translator now take missing_translation_behavior. behavior still works as a keyword-only deprecated alias that raises a DeprecationWarning, and when both are given missing_translation_behavior wins. Neither given means Original, as before. A value given by position is taken as missing_translation_behavior, as it was taken as behavior. The IP Intelligence translation engines now pass every option by its name, so they need a fiftyone_pipeline_translation that takes the new name. Their package requires fiftyone_pipeline_translation 4.5.43 or later, the version the release plan gives this commit. * Leave the IP Intelligence translation engines as they were on rc The IP Intelligence translation engines pass the missing translation behavior by position. A value passed by position is taken as missing_translation_behavior by the new translation package and as behavior by the old one, so the engines work with either release and raise no deprecation warning without any change. Passing it by name needed a minimum version of fiftyone_pipeline_translation that cannot be known until the release that carries the new name, so the engines and their package's requirements are back to their rc content. * Take None as not given for the missing translation behavior rc treated None as Original, because the translator took any value that was not EMPTY_STRING or FLOW_ERROR as Original. Refusing it would stop a configuration holding null, or code passing an unset option on as None, from building after a patch release. The repository's other engines use None to mean not set, so the translation engine and Translator now do the same. Both parameters default to None, None under either name gives Original without a warning, and anything else that names no member is still refused. --- .../missing_translation_behavior.py | 54 ++++ .../translation_engine_base.py | 22 +- .../translator.py | 22 +- .../tests/test_translation.py | 265 ++++++++++++++++++ 4 files changed, 354 insertions(+), 9 deletions(-) diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py index 794ec987..619f843b 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/missing_translation_behavior.py @@ -20,6 +20,7 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +import warnings from enum import Enum @@ -38,3 +39,56 @@ class MissingTranslationBehavior(Enum): # Add a not-found error to the flow data and return None. FLOW_ERROR = "flow_error" + + @classmethod + def from_value(cls, value): + + """! + The member a value names. A member is returned as it is. Text names + a member by the specification's name (Original, EmptyString or + FlowError) or by the member's own name or value, ignoring case and + underscores, because a configuration file cannot give a member. + + @param value: A member, or text naming one. + + @rtype: MissingTranslationBehavior + @return: The member named. + + @raises ValueError: When the value names no member. + """ + + if isinstance(value, cls): + return value + if isinstance(value, str): + key = value.replace("_", "").lower() + for member in cls: + if member.name.replace("_", "").lower() == key: + return member + raise ValueError( + f"{value!r} is not a missing translation behavior. Use " + f"Original, EmptyString or FlowError.") + + +def _resolve(missing_translation_behavior, behavior): + + """! + The member the constructor arguments name. None means not given, as for + the other engines' optional settings, so a setting passed on as None + gives ORIGINAL. behavior is the deprecated name for + missing_translation_behavior, so using it warns, and when both are given + missing_translation_behavior wins. Neither given means ORIGINAL. + """ + + if behavior is not None: + # Two levels up is the caller of the constructor that called this. + warnings.warn( + "The 'behavior' argument is deprecated. Use " + "'missing_translation_behavior', the name the specification " + "gives the option.", + DeprecationWarning, + stacklevel=3) + if missing_translation_behavior is None: + missing_translation_behavior = behavior + if missing_translation_behavior is None: + return MissingTranslationBehavior.ORIGINAL + return MissingTranslationBehavior.from_value(missing_translation_behavior) diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py index 4f349e76..8be1d175 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translation_engine_base.py @@ -31,7 +31,8 @@ from .languages import Languages, validate_locale from .translator import Translator -from .missing_translation_behavior import MissingTranslationBehavior +from .missing_translation_behavior import ( + MissingTranslationBehavior, _resolve) class TranslationEngineBase(FlowElement): @@ -65,7 +66,9 @@ def __init__( translations, sources, fixed_language=None, - behavior=MissingTranslationBehavior.ORIGINAL): + missing_translation_behavior=None, + *, + behavior=None): """! Constructor. @@ -86,8 +89,17 @@ def __init__( always translates to this language, otherwise the language is resolved from the evidence. - @type behavior: MissingTranslationBehavior - @param behavior: Behavior when a translation is missing for a value. + @type missing_translation_behavior: MissingTranslationBehavior or + string + @param missing_translation_behavior: Behavior when a translation is + missing for a value, as a member or as text naming one, such as + "EmptyString". ORIGINAL when not given or None. Any other value + raises ValueError. + + @type behavior: MissingTranslationBehavior or string + @param behavior: Deprecated name for missing_translation_behavior, + which raises a DeprecationWarning when used. When both are given, + missing_translation_behavior wins. """ super().__init__() @@ -100,6 +112,8 @@ def __init__( if not source_element_data_key or not source_element_data_key.strip(): raise ValueError("The source element key must be configured.") + behavior = _resolve(missing_translation_behavior, behavior) + self.datakey = "translation" self.source_element_data_key = source_element_data_key.strip() self._behavior = behavior diff --git a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py index 19be2cd3..0a014c25 100644 --- a/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py +++ b/pipeline/fiftyone_pipeline_translation/src/fiftyone_pipeline_translation/translator.py @@ -22,7 +22,8 @@ from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue -from .missing_translation_behavior import MissingTranslationBehavior +from .missing_translation_behavior import ( + MissingTranslationBehavior, _resolve) class Translator: @@ -41,7 +42,9 @@ class Translator: def __init__( self, translations=None, - behavior=MissingTranslationBehavior.ORIGINAL): + missing_translation_behavior=None, + *, + behavior=None): """! Constructor. @@ -50,15 +53,24 @@ def __init__( @param translations: Map of source value to translated value. Lookups are case-insensitive. - @type behavior: MissingTranslationBehavior - @param behavior: Behavior when a translation is missing for a value. + @type missing_translation_behavior: MissingTranslationBehavior or + string + @param missing_translation_behavior: Behavior when a translation is + missing for a value, as a member or as text naming one, such as + "EmptyString". ORIGINAL when not given or None. Any other value + raises ValueError. + + @type behavior: MissingTranslationBehavior or string + @param behavior: Deprecated name for missing_translation_behavior, + which raises a DeprecationWarning when used. When both are given, + missing_translation_behavior wins. """ translations = translations or {} # Store with lower-cased keys so lookups are case-insensitive. self._translations = { str(key).lower(): value for key, value in translations.items()} - self._behavior = behavior + self._behavior = _resolve(missing_translation_behavior, behavior) def translate(self, value, errors=None): diff --git a/pipeline/fiftyone_pipeline_translation/tests/test_translation.py b/pipeline/fiftyone_pipeline_translation/tests/test_translation.py index ebafc0d9..644e23b9 100644 --- a/pipeline/fiftyone_pipeline_translation/tests/test_translation.py +++ b/pipeline/fiftyone_pipeline_translation/tests/test_translation.py @@ -21,6 +21,7 @@ # ********************************************************************* import unittest +import warnings from fiftyone_pipeline_core.flowelement import FlowElement from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary @@ -183,5 +184,269 @@ def test_locale_from_filename(self): TranslationEngineBase._get_language_name("countries.fr_FR.yml")) +# Each text the missing translation behavior accepts, and the member it +# names. The specification's names, the members' names and values, and +# other mixes of case and underscores are all accepted. +BEHAVIOR_TEXTS = { + "Original": MissingTranslationBehavior.ORIGINAL, + "original": MissingTranslationBehavior.ORIGINAL, + "ORIGINAL": MissingTranslationBehavior.ORIGINAL, + "EmptyString": MissingTranslationBehavior.EMPTY_STRING, + "emptystring": MissingTranslationBehavior.EMPTY_STRING, + "empty_string": MissingTranslationBehavior.EMPTY_STRING, + "EMPTY_STRING": MissingTranslationBehavior.EMPTY_STRING, + "Empty_String": MissingTranslationBehavior.EMPTY_STRING, + "FlowError": MissingTranslationBehavior.FLOW_ERROR, + "flowerror": MissingTranslationBehavior.FLOW_ERROR, + "flow_error": MissingTranslationBehavior.FLOW_ERROR, + "FLOW_ERROR": MissingTranslationBehavior.FLOW_ERROR, +} + +# Values that name no member, each refused when the engine is built. +UNKNOWN_BEHAVIORS = ( + "", "Empty String", "Origin", "Missing", "flow-error", 1, + ["Original"]) + +# The specification's name for each member. +SPECIFICATION_NAMES = { + "Original": MissingTranslationBehavior.ORIGINAL, + "EmptyString": MissingTranslationBehavior.EMPTY_STRING, + "FlowError": MissingTranslationBehavior.FLOW_ERROR, +} + +# The value a missing translation of "Spain" gives under each member, with +# the number of flow errors it adds. +MISSING_RESULTS = { + MissingTranslationBehavior.ORIGINAL: (["Allemagne", "Spain"], 0), + MissingTranslationBehavior.EMPTY_STRING: (["Allemagne", ""], 0), + MissingTranslationBehavior.FLOW_ERROR: (["Allemagne", None], 1), +} + + +class MissingTranslationBehaviorTests(unittest.TestCase): + + """ + The missing translation behavior given as a member or as text, as a + configuration file gives it, with anything else refused when the engine + is built, apart from None, which means not given. The option takes the + specification's name, missing_translation_behavior, and still takes the + old name, behavior, with a DeprecationWarning. + """ + + def _engine(self, behavior): + return TranslationEngineBase( + "source", + [("Names", "NamesTranslated")], + FR_SOURCE, + "fr_FR", + behavior) + + def _missing(self, pipeline): + """The translated names and the flow errors for a source holding a + name with a translation and one without. The pipeline keeps errors + on the flow data rather than raising them, so a flow error can be + counted.""" + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.process() + translated = flowdata.get("translation").get("NamesTranslated") + if isinstance(translated, AspectPropertyValue): + translated = translated.value() + return translated, len(flowdata.errors) + + def _pipeline(self, behavior): + return self._pipeline_with(self._engine(behavior)) + + @staticmethod + def _pipeline_with(engine): + source = MockSourceElement("source", { + "Names": AspectPropertyValue(value=["Germany", "Spain"]) + }) + return PipelineBuilder({"suppress_process_exceptions": True}).add( + source).add(engine).build() + + @staticmethod + def _engine_with(**options): + """An engine built with every option given by its name.""" + return TranslationEngineBase( + source_element_data_key="source", + translations=[("Names", "NamesTranslated")], + sources=FR_SOURCE, + fixed_language="fr_FR", + **options) + + def test_from_value_names_each_member(self): + for text, member in BEHAVIOR_TEXTS.items(): + with self.subTest(text): + self.assertIs( + member, MissingTranslationBehavior.from_value(text)) + + def test_from_value_returns_a_member_as_it_is(self): + for member in MissingTranslationBehavior: + with self.subTest(member.name): + self.assertIs( + member, MissingTranslationBehavior.from_value(member)) + + def test_each_text_takes_effect(self): + for text, member in BEHAVIOR_TEXTS.items(): + with self.subTest(text): + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline(text))) + + def test_each_member_takes_effect(self): + for member, expected in MISSING_RESULTS.items(): + with self.subTest(member.name): + self.assertEqual( + expected, self._missing(self._pipeline(member))) + + def test_unknown_value_is_refused_when_the_engine_is_built(self): + for value in UNKNOWN_BEHAVIORS: + with self.subTest(repr(value)): + with self.assertRaises(ValueError) as raised: + self._engine(value) + self.assertIn("EmptyString", str(raised.exception)) + + def test_translator_takes_text(self): + translator = Translator({"Germany": "Allemagne"}, "EmptyString") + self.assertEqual("", translator.translate("Spain")) + + def test_specification_name_takes_effect_without_a_warning(self): + for text, member in SPECIFICATION_NAMES.items(): + with self.subTest(text): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + engine = self._engine_with( + missing_translation_behavior=text) + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline_with(engine))) + + def test_old_name_still_takes_effect_with_a_warning(self): + for text, member in SPECIFICATION_NAMES.items(): + with self.subTest(text): + with self.assertWarns(DeprecationWarning) as warned: + engine = self._engine_with(behavior=text) + self.assertIn( + "missing_translation_behavior", str(warned.warning)) + self.assertEqual( + MISSING_RESULTS[member], + self._missing(self._pipeline_with(engine))) + + def test_specification_name_wins_when_both_are_given(self): + with self.assertWarns(DeprecationWarning): + engine = self._engine_with( + missing_translation_behavior="EmptyString", + behavior="FlowError") + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.EMPTY_STRING], + self._missing(self._pipeline_with(engine))) + + def test_neither_name_given_is_original(self): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + engine = self._engine_with() + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.ORIGINAL], + self._missing(self._pipeline_with(engine))) + + def test_none_given_by_either_name_is_original(self): + # None means not given, so it gives Original without a warning. + translations = {"Germany": "Allemagne"} + for given in ("by position", "missing_translation_behavior", + "behavior"): + with self.subTest(given): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + if given == "by position": + engine = self._engine(None) + translator = Translator(translations, None) + else: + engine = self._engine_with(**{given: None}) + translator = Translator(translations, **{given: None}) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.ORIGINAL], + self._missing(self._pipeline_with(engine))) + self.assertEqual("Spain", translator.translate("Spain")) + + def test_translator_takes_both_names(self): + translations = {"Germany": "Allemagne"} + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + translator = Translator( + translations, missing_translation_behavior="EmptyString") + self.assertEqual("", translator.translate("Spain")) + with self.assertWarns(DeprecationWarning): + translator = Translator(translations, behavior="EmptyString") + self.assertEqual("", translator.translate("Spain")) + with self.assertWarns(DeprecationWarning): + translator = Translator( + translations, + missing_translation_behavior="Original", + behavior="EmptyString") + self.assertEqual("Spain", translator.translate("Spain")) + + def test_configuration_text_takes_effect(self): + for text, member in (("EmptyString", "EMPTY_STRING"), + ("flow_error", "FLOW_ERROR"), + ("Original", "ORIGINAL")): + with self.subTest(text): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + pipeline = PipelineBuilder().build_from_configuration( + self._configuration(text)) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior[member]], + self._missing(pipeline)) + + def test_configuration_with_an_unknown_value_is_refused(self): + with self.assertRaises(ValueError): + PipelineBuilder().build_from_configuration( + self._configuration("EmptyStrings")) + + def test_configuration_with_the_old_name_still_takes_effect(self): + with self.assertWarns(DeprecationWarning): + pipeline = PipelineBuilder().build_from_configuration( + self._configuration("EmptyString", "behavior")) + self.assertEqual( + MISSING_RESULTS[MissingTranslationBehavior.EMPTY_STRING], + self._missing(pipeline)) + + @staticmethod + def _configuration(behavior, name="missing_translation_behavior"): + """A pipeline configuration as a configuration file gives one, so + every value in it is text, a list or a dictionary. Every option has + the specification's name, unless the behavior's name is given.""" + return { + "PipelineOptions": { + "suppress_process_exceptions": True, + "Elements": [ + { + "elementPath": __name__, + "elementName": "MockSourceElement", + "elementParameters": { + "datakey": "source", + "contents": {"Names": ["Germany", "Spain"]}, + }, + }, + { + "elementPath": ( + "fiftyone_pipeline_translation." + "translation_engine_base"), + "elementName": "TranslationEngineBase", + "elementParameters": { + "source_element_data_key": "source", + "translations": [["Names", "NamesTranslated"]], + "sources": FR_SOURCE, + "fixed_language": "fr_FR", + name: behavior, + }, + }, + ] + } + } + + if __name__ == "__main__": unittest.main() From 88be76bfa6ea414de6f2373235d8bce19e18479f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 14:45:51 +0000 Subject: [PATCH 23/26] Package text says what each package does in the words people search for (#68) Name the competitor brands people search for in the remaining package text. Text only, no code change. --- .../fiftyone_devicedetection_shared/setup.py | 15 ++++++++++++++- .../fiftyone_ipintelligence_shared/setup.py | 13 ++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/device-detection/fiftyone_devicedetection_shared/setup.py b/device-detection/fiftyone_devicedetection_shared/setup.py index 07b89cdf..c57092a3 100644 --- a/device-detection/fiftyone_devicedetection_shared/setup.py +++ b/device-detection/fiftyone_devicedetection_shared/setup.py @@ -46,7 +46,20 @@ def read(file_name): "Documentation": "https://51degrees.com/documentation/_device_detection__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_shared&utm_term=documentation", "Source": "https://github.com/51Degrees/51degrees-python", }, - description=("The 51Degrees Pipeline API is a generic web request intelligence and data processing solution with the ability to add a range of 51Degrees and/or custom plug ins (Engines). This project contains 51Degrees Device Detection engines that can be used with the Pipeline API."), + description=("Device Detection and User-Agent parser for Python that detects the device, browser, operating system, and crawlers from the User-Agent and User-Agent Client Hints. This package holds the code shared by the cloud and on-premise engines for the 51Degrees Pipeline API. An alternative to UAParser.js, DeviceAtlas, and WURFL."), + keywords=[ + "51degrees", + "device-detection", + "user-agent", + "user-agent-parser", + "client-hints", + "browser-detection", + "mobile-detection", + "crawler-detection", + "wurfl", + "deviceatlas", + "uaparser" + ], long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.10", diff --git a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py index 7f89e1d0..a4c4934c 100644 --- a/ip-intelligence/fiftyone_ipintelligence_shared/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_shared/setup.py @@ -45,7 +45,18 @@ def read(file_name): "Documentation": "https://51degrees.com/documentation/_ip_intelligence__index.html?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_shared&utm_term=documentation", "Source": "https://github.com/51Degrees/51degrees-python", }, - description=("Shared code used by the cloud and on-premise variants of the 51Degrees IP Intelligence engines for the 51Degrees Pipeline API."), + description=("IP Intelligence and IP geolocation for Python that finds an IP address's likely location (country, region, town, and coordinates), connection type, and network. This package holds the code shared by the cloud and on-premise engines for the 51Degrees Pipeline API. An alternative to MaxMind and IPinfo."), + keywords=[ + "51degrees", + "ip-intelligence", + "ip-geolocation", + "geoip", + "ip-lookup", + "asn", + "ip-to-country", + "maxmind", + "ipinfo" + ], long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.10", From a474f045675a890896f1c7cfc328e2b438412b77 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 15:15:29 +0000 Subject: [PATCH 24/26] Check the callback URL as the specification's six rules say (#66) * Check the callback URL as the specification's six rules say Refuse an IPv6 zone identifier, an '@', '?' or '#' anywhere in the host value, and a name ending in a number unless it is an IPv4 address in dotted decimal form. The logged URL shows *** in place of everything in the host before its last '@'. * Test that a port which is present is not empty * Refuse a port that is present but empty A host such as https://example.com otherwise reads as the host https with an empty port. RFC 3986 section 3.2.3 allows an empty port, but section 6.2.3 treats it as equal to none, so the callback URL does not carry one. --- .../fiftyone_pipeline_core/callback_url.py | 79 ++++----- .../tests/test_callback_url.py | 151 +++++++++++++++--- .../tests/test_javascriptbuilder_variables.py | 45 ++++++ 3 files changed, 206 insertions(+), 69 deletions(-) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py index b18fbb13..15854691 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/callback_url.py @@ -25,9 +25,9 @@ and host usually come from the request, and the script holds the URL in a single-quoted string, so the URL is checked before it is written into the script. A URL that is not valid, including one whose protocol is not http or -https, is refused. In the user information, in anything after the host and -port, and in the endpoint, the characters a URL cannot hold or that could end -the string or the script are percent-encoded. +https, is refused. In anything after the authority, and in the endpoint, the +characters a URL cannot hold or that could end the string or the script are +percent-encoded. See https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/javascript-builder.md#callback-url """ @@ -67,19 +67,18 @@ _ASCII_LOWER = str.maketrans( "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") -# Where the authority ends, which is at the first '/', '?' or '#' of the -# host. -AUTHORITY_END_PATTERN = re.compile(r"[/?#]") +# The characters the host value must not hold. +FORBIDDEN_PATTERN = re.compile(r"[?#@]") # A label of a host name, being ASCII letters, digits, '-' and '_'. LABEL_PATTERN = re.compile(r"[A-Za-z0-9_-]+") -# An IPv6 zone identifier, being unreserved characters as RFC 3986 section -# 2.3 defines them. -ZONE_PATTERN = re.compile(r"[A-Za-z0-9._~-]+") +# A last label that makes the host a number, being only digits, or 0x or 0X +# and any number of hexadecimal digits. +NUMBER_LABEL_PATTERN = re.compile(r"[0-9]+|0[xX][0-9A-Fa-f]*") -# A port as it follows the host, being ':' and any number of digits. -PORT_PATTERN = re.compile(r":[0-9]*") +# A port as it follows the host, being ':' and at least one digit. +PORT_PATTERN = re.compile(r":[0-9]+") # One group of an IPv6 address. HEX_GROUP_PATTERN = re.compile(r"[0-9A-Fa-f]{1,4}") @@ -147,18 +146,19 @@ def callback_url(protocol, host, endpoint): Build the callback URL from its parts and check it. One slash separates the host and the endpoint, being added when neither has one and removed when both do. The protocol and the host are written in lower case and - the rest as given, apart from the percent-encoding. + the rest as given, apart from the percent-encoding. The host value holds + no '?', '#' or '@'. @type protocol: str @param protocol: The protocol, http or https in any case @type host: str - @param host: The host, which can start with user information and end - with a port, a path, a query or a fragment + @param host: The host, which can end with a port and a path @type endpoint: str @param endpoint: The endpoint @rtype: CallbackUrl @return: The callback URL, or None when the protocol, host or endpoint - is not set + is not set. The URL of an invalid one is as it was built, for the log, + with '***' in place of everything in the host before its last '@'. """ if is_blank(protocol) or is_blank(host) or is_blank(endpoint): @@ -169,12 +169,14 @@ def callback_url(protocol, host, endpoint): endpoint = "/" + endpoint elif host_has_slash and endpoint_has_slash: endpoint = endpoint[1:] - built = protocol + "://" + host + endpoint - if not is_scheme(protocol): + at = host.rfind("@") + built = (protocol + "://" + ("***" + host[at:] if at != -1 else host) + + endpoint) + if not is_scheme(protocol) or FORBIDDEN_PATTERN.search(host): return CallbackUrl(False, built) - match = AUTHORITY_END_PATTERN.search(host) - end = len(host) if match is None else match.start() - authority = _checked_authority(host[:end]) + end = host.find("/") + end = len(host) if end == -1 else end + authority = _checked_host_port(host[:end]) if authority is None: return CallbackUrl(False, built) return CallbackUrl( @@ -183,40 +185,21 @@ def callback_url(protocol, host, endpoint): + _encoded(host[end:]) + _encoded(endpoint)) -def _checked_authority(authority): - - """! - The user information, host and port as they are written into the URL, - or None when they are not valid. The user information ends at the first - '@', so a second '@' is part of the host and refused. - """ - - user, at, host_port = authority.partition("@") - if not at: - return _checked_host_port(authority) - host_port = _checked_host_port(host_port) - if host_port is None: - return None - return _encoded(user) + "@" + host_port - - def _checked_host_port(host_port): """! The host and optional port in lower case, or None when they are not - valid. The host is an IPv6 address in brackets with an optional zone - identifier, or a name of labels separated by dots with a dot allowed at - the end. The port is ':' followed by digits up to 65535, or by nothing. + valid. The host is an IPv6 address in brackets, or a name of labels + separated by dots with a dot allowed at the end, which is an IPv4 address + when its last label is a number. The port is ':' followed by digits up to + 65535. """ if host_port.startswith("["): close = host_port.find("]") if close == -1: return None - address, separator, zone = host_port[1:close].partition("%25") - if separator and ZONE_PATTERN.fullmatch(zone) is None: - return None - if not _is_ipv6(address): + if not _is_ipv6(host_port[1:close]): return None host = host_port[:close + 1] port = host_port[close + 1:] @@ -224,9 +207,13 @@ def _checked_host_port(host_port): host, colon, port = host_port.partition(":") port = colon + port name = host[:-1] if host.endswith(".") else host - for label in name.split("."): + labels = name.split(".") + for label in labels: if LABEL_PATTERN.fullmatch(label) is None: return None + if (NUMBER_LABEL_PATTERN.fullmatch(labels[-1]) + and IPV4_PATTERN.fullmatch(host) is None): + return None if port and not _is_port(port): return None return host.lower() + port @@ -236,7 +223,7 @@ def _is_port(text): """! Whether the text is ':' followed by digits whose value is no more than - 65535, or by nothing. + 65535. """ if PORT_PATTERN.fullmatch(text) is None: diff --git a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py index 129fee0d..29e8be4d 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_callback_url.py @@ -69,21 +69,22 @@ def test_missing_when_a_part_is_empty(self): @parameterized.expand([ ("EXAMPLE.com", "https://example.com/51dpipeline/json"), ("example.com:8080", "https://example.com:8080/51dpipeline/json"), - ("example.com:", "https://example.com:/51dpipeline/json"), ("example.com:65535", "https://example.com:65535/51dpipeline/json"), ("a_b.com", "https://a_b.com/51dpipeline/json"), ("-ab.com", "https://-ab.com/51dpipeline/json"), ("127.0.0.1:80", "https://127.0.0.1:80/51dpipeline/json"), + ("1.2.3.4", "https://1.2.3.4/51dpipeline/json"), + ("1.2.3.4:8080", "https://1.2.3.4:8080/51dpipeline/json"), + ("1.2.3.4/path", "https://1.2.3.4/path/51dpipeline/json"), + ("0.0.0.0", "https://0.0.0.0/51dpipeline/json"), + ("101.2.3.4", "https://101.2.3.4/51dpipeline/json"), + ("example.co1", "https://example.co1/51dpipeline/json"), + ("123.example.com", "https://123.example.com/51dpipeline/json"), + ("0x7f.example.com", "https://0x7f.example.com/51dpipeline/json"), ("[::1]", "https://[::1]/51dpipeline/json"), ("[::1]:8080", "https://[::1]:8080/51dpipeline/json"), - ("[fe80::1%25eth0]", "https://[fe80::1%25eth0]/51dpipeline/json"), - ("user:pw@example.com", - "https://user:pw@example.com/51dpipeline/json"), - ("u x@example.com", "https://u%20x@example.com/51dpipeline/json"), - ("u\tx@example.com", "https://u%09x@example.com/51dpipeline/json"), ("example.com.", "https://example.com./51dpipeline/json"), ("example.com/path", "https://example.com/path/51dpipeline/json"), - ("example.com?x", "https://example.com?x/51dpipeline/json"), ("example.com/a b", "https://example.com/a%20b/51dpipeline/json"), ("example.com/a%zzb", "https://example.com/a%25zzb/51dpipeline/json"), @@ -110,7 +111,6 @@ def test_accepts_the_host(self, host, expected): ("[::1",), ("[v1.x]",), ("[1.2.3.4]",), - ("a@b@example.com",), ("a..b.com",), ("%41.com",), ("a\\b.com",), @@ -155,11 +155,6 @@ def test_refuses_the_zone_or_the_text_after_the_bracket(self, host): self.assertFalse(_url("https", host).valid, repr(host)) - def test_accepts_a_zone_followed_by_a_port(self): - self.assertEqual( - _valid("https://[fe80::1%25eth0]:8080/51dpipeline/json"), - _url("https", "[fe80::1%25eth0]:8080")) - def test_checks_the_port_by_its_value(self): """A port goes by its value, so leading zeros are allowed, and a port too long to read is refused rather than read as small enough.""" @@ -178,15 +173,14 @@ def test_checks_the_port_by_its_value(self): self.assertFalse(_url("https", host).valid, host) def test_encodes_the_characters_rule_8_lists(self): - """Each character rule 8 lists, in the user information, after the - host and in the endpoint, and a '%' without two hexadecimal digits - after it, even at the end.""" + """Each character rule 8 lists, after the host and in the endpoint, + and a '%' without two hexadecimal digits after it, even at the + end.""" self.assertEqual( - _valid("https://u%22%5E%60%7B%7C%7D@example.com/%22%5E%60%7B%7C%7D" + _valid("https://example.com/%22%5E%60%7B%7C%7D" "/j%22%5E%60%7B%7C%7D"), - callback_url( - "https", 'u"^`{|}@example.com/"^`{|}', '/j"^`{|}')) + callback_url("https", 'example.com/"^`{|}', '/j"^`{|}')) self.assertEqual( _valid("https://example.com/a%25/51dpipeline/json"), _url("https", "example.com/a%")) @@ -200,6 +194,120 @@ def test_encodes_the_characters_rule_8_lists(self): _valid("https://example.com/json%25"), callback_url("https", "example.com", "/json%")) + @parameterized.expand([ + ("[fe80::1%25eth0]",), + ("[fe80::1%25eth0]:8080",), + ("[FE80::1%25ETH0]",), + ("[fe80::1%251]",), + ("[::1%25lo]",), + ("[fe80::1%]",), + ("[fe80::1%25eth0]/path",), + ]) + def test_refuses_the_zone_identifier(self, host): + """Rule 4 refuses an IPv6 zone identifier.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("user@example.com",), + ("user:pw@example.com",), + ("@example.com",), + (":@example.com",), + ("user@example.com:8080",), + ("user@[::1]",), + ("user@example.com/path",), + ("u%41@example.com",), + ("u x@example.com",), + ("u\tx@example.com",), + ("u\"x@example.com",), + ("u'x@example.com",), + ("u\\x@example.com",), + ("u\nx@example.com",), + ("u\u0001x@example.com",), + ("a@b@example.com",), + ("example.com/a@b",), + ("exa mple.com/a@b",), + ("user:12/ss@example.com",), + ("user:pa/ss@example.com",), + ("user:pa?ss@example.com",), + ]) + def test_refuses_the_user_information(self, host): + """Rule 3 refuses an '@' anywhere in the host value.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.com?x",), + ("example.com#x",), + ("example.com?",), + ("example.com#",), + ("example.com:8080?x",), + ("[::1]#x",), + ("example.com/path?x",), + ("example.com/path#x",), + ("example.com?');alert(1);//",), + ("example.com#');alert(1);//",), + ]) + def test_refuses_the_query_or_fragment(self, host): + """Rule 2 refuses a '?' or '#' anywhere in the host value.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.123",), + ("example.0x1f",), + ("example.0X",), + ("1.2.3.999",), + ("1.2.3",), + ("127.1",), + ("0x7f.0.0.1",), + ("01.2.3.4",), + ("00.2.3.4",), + ("example.123.",), + ("1.2.3.4.",), + ("1.2.3.4.5",), + ("1",), + ("300.1.1.1",), + ("0x7f.1",), + ("1.2.3.4.:8080",), + ]) + def test_refuses_a_name_ending_in_a_number(self, host): + """Rule 4 reads a name whose last label is a number as an IPv4 + address written as four decimal numbers, each starting with 0 only + when it is 0 alone.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("example.com:",), + ("example.com:/path",), + ("[::1]:",), + ("https://example.com",), + ("http://example.com",), + ]) + def test_refuses_the_empty_port(self, host): + """Rule 5 refuses a port that is present but empty, so a protocol + written into the host gives the host "https" with an empty port.""" + + self.assertFalse(_url("https", host).valid, repr(host)) + + @parameterized.expand([ + ("user:secret@example.com", + "https://***@example.com/51dpipeline/json"), + ("a@b@example.com", "https://***@example.com/51dpipeline/json"), + ("u@[::1]:80x", "https://***@[::1]:80x/51dpipeline/json"), + ("user:pa/ss@example.com", + "https://***@example.com/51dpipeline/json"), + ("user:pa?ss@example.com", + "https://***@example.com/51dpipeline/json"), + ("exa mple.com/a@b", "https://***@b/51dpipeline/json"), + ]) + def test_the_logged_url_holds_no_user_information(self, host, logged): + """Rule 6 shows *** in place of everything in the host before its + last '@', wherever in the host value that '@' is.""" + + self.assertEqual(CallbackUrl(False, logged), _url("https", host)) + def test_the_invalid_url_is_the_one_built(self): self.assertEqual( CallbackUrl(False, "https://exa mple.com/51dpipeline/json"), @@ -224,10 +332,7 @@ def test_checks_the_scheme(self): @parameterized.expand([ ("example.com/');alert(1);//",), - ("example.com?');alert(1);//",), - ("example.com#');alert(1);//",), - ("u'x@example.com",), - ("u\\x@example.com",), + ('example.com/a"b',), ("example.com/a\\b",), ("example.com/",), ]) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py index 7b601e61..9ac0e771 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_variables.py @@ -202,6 +202,51 @@ def test_refused_url_is_left_out_and_logged_once(self): self.assertEqual(1, len(errors), errors) self.assertIn("'https://exa mple.com/json'", errors[0]) + @parameterized.expand([ + ("user:secret@example.com", "'https://***@example.com/json'"), + ("a@b@example.com", "'https://***@example.com/json'"), + ("u@[::1]:80x", "'https://***@[::1]:80x/json'"), + ("user:pa/ss@example.com", "'https://***@example.com/json'"), + ("user:pa?ss@example.com", "'https://***@example.com/json'"), + ("exa mple.com/a@b", "'https://***@b/json'"), + ]) + def test_refused_host_is_logged_without_its_user_information( + self, host, logged): + """The log never holds what comes before the last '@' of the host + value. Each host has an element of its own because a refusal is + logged once per element.""" + + rendered, errors = _render_with_hosts([host]) + + self.assertNotIn("_url", rendered[0]) + self.assertEqual(1, len(errors), errors) + self.assertIn(logged, errors[0]) + self.assertNotIn("secret", errors[0]) + self.assertNotIn("'pa", errors[0]) + self.assertNotIn("pa/", errors[0]) + + def test_refused_configured_host_is_logged_without_user_information( + self): + """A host from the settings is refused and logged the same way.""" + + logger = MemoryLogger("error") + pipeline = PipelineBuilder({"javascript_builder_settings": { + "minify": False, "endpoint": "/json", + "host": "user:secret@example.com"}}).add_logger( + logger).build() + captured = _CapturedAll() + with mock.patch.object( + javascriptbuilder.chevron, "render", captured.render): + pipeline.create_flowdata().process() + errors = [ + entry["message"] for entry in logger.memory_log + if entry["level"] == "error"] + + self.assertNotIn("_url", captured.variables[0]) + self.assertEqual(1, len(errors), errors) + self.assertIn("'https://***@example.com/json'", errors[0]) + self.assertNotIn("secret", errors[0]) + def test_refused_url_is_logged_on_one_line(self): """The URL is quoted and escaped in the log, so a line break in it cannot start a new log line.""" From 147ed13c37790484552c370c448c1a526a582aca Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:20:20 +0000 Subject: [PATCH 25/26] Refuse a 51Did text with white space inside it, or both base64 alphabets (#56) * Refuse a 51Did text with white space inside it, or both base64 alphabets The 51Did specification says a reader ignores white space before and after the text, being any character with the Unicode White_Space property, refuses any other character outside the alphabet, white space inside the text included, and refuses a text holding a character from each alphabet, checking that before it maps "-" and "_" onto "+" and "/". FodId.to_standard_base64 now removes exactly the White_Space characters around the text, and answers None for a text holding any other character outside the two alphabets and the padding, or holding a character from each alphabet. The base64 readers report InvalidBase64 for such a text. White space alone still reports MissingInput. Before this change the reader read a mixed text as if it were clean, and read a text with U+001C to U+001F around it, because str.strip() removes those characters although they are not White_Space. White space inside the text was already refused by the OWID decoder. The client's length check and to_base64_url trim the same characters. * Say the alphabet check is made separately in the fod_id comment --- pipeline/fiftyone_pipeline_did/readme.md | 15 +- .../src/fiftyone_pipeline_did/did_client.py | 10 +- .../src/fiftyone_pipeline_did/fod_id.py | 99 +++++++++-- .../fiftyone_pipeline_did/tests/test_fodid.py | 162 ++++++++++++++++++ 4 files changed, 258 insertions(+), 28 deletions(-) diff --git a/pipeline/fiftyone_pipeline_did/readme.md b/pipeline/fiftyone_pipeline_did/readme.md index fe51f236..3c2f369c 100644 --- a/pipeline/fiftyone_pipeline_did/readme.md +++ b/pipeline/fiftyone_pipeline_did/readme.md @@ -112,11 +112,12 @@ url_safe = fod_id.as_base64_url() # URL-safe alphabet, no padding, for a link ``` `from_base64` accepts the standard alphabet the cloud issues and the -URL-safe alphabet a page puts in a link, with or without padding. On an -identifier carrying a creator context the License Id field holds an -encrypted value that only 51Degrees can turn back into a licence -identifier, so `license_id` is the field's raw value and identifies -nothing outside 51Degrees. +URL-safe alphabet a page puts in a link, with or without padding. White +space around the value is ignored. A value with white space inside it, or +with characters from both alphabets, is refused. On an identifier carrying +a creator context the License Id field holds an encrypted value that only +51Degrees can turn back into a licence identifier, so `license_id` is the +field's raw value and identifies nothing outside 51Degrees. ### The usage an identifier was created for @@ -267,9 +268,9 @@ same whichever language parsed the bytes. | Status | Meaning | | --- | --- | | `PARSED` | A structurally valid 51Did. The signature has not been checked | -| `MISSING_INPUT` | `None`, an empty string or an empty buffer | +| `MISSING_INPUT` | `None`, an empty string, a string of white space alone or an empty buffer | | `INVALID_INPUT_TYPE` | Not a string (base64 reader) or not a bytes-like object (byte reader) | -| `INVALID_BASE64` | The text is not base64 in either alphabet | +| `INVALID_BASE64` | The text is not base64 in one alphabet, so there are no bytes. White space inside the text, and characters from both alphabets, report this | | `UNSUPPORTED_VERSION` | The first byte names an envelope version this package does not know | | `UNEXPECTED_END` | The data stopped in the middle of an envelope field | | `INVALID_DOMAIN_ENCODING` | The creator domain is not terminated or is longer than the OWID maximum | diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 6297b286..1a28e810 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -65,7 +65,7 @@ from ._owid import Version from ._layout import GUID_LENGTH, HEADER_LENGTH, MATCH_KEY_LENGTH -from .fod_id import DATE_EPOCH, FodId, _date_minutes +from .fod_id import DATE_EPOCH, FodId, _date_minutes, _trim_white_space from .id_type import IdType #: The public cloud API base, used when neither the ``endpoint`` argument @@ -934,10 +934,10 @@ def _ensure_parses(value: str) -> None: def _ensure_encoded_size(value: str) -> None: """Refuses obviously malformed text before it is parsed, a key is - fetched or the cloud is called. Surrounding whitespace is stripped - before measuring, as the reader strips it before decoding, so the two - measure the same characters.""" - if len(value.strip()) > _MAXIMUM_ENCODED_LENGTH: + fetched or the cloud is called. White space around the value is + removed before measuring, as the reader removes it before decoding, + so the two measure the same characters.""" + if len(_trim_white_space(value)) > _MAXIMUM_ENCODED_LENGTH: raise ValueError( "The identifier is far longer than any 51Did the cloud " "issues.") diff --git a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index a71e388f..0953b187 100644 --- a/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/pipeline/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -22,6 +22,7 @@ from __future__ import annotations +import re from datetime import datetime, timezone from enum import Enum from typing import NamedTuple, Optional, Tuple @@ -82,7 +83,9 @@ class FodIdParseStatus(Enum): MISSING_INPUT = "MissingInput" #: The input was supplied in a form this surface cannot read. INVALID_INPUT_TYPE = "InvalidInputType" - #: The string is not valid base 64, so there are no bytes to read. + #: The string is not valid base 64 in one alphabet, so there are no + #: bytes to read. White space inside the string, and characters from + #: both alphabets, report this. INVALID_BASE64 = "InvalidBase64" #: The first byte names an envelope version this package does not know. UNSUPPORTED_VERSION = "UnsupportedVersion" @@ -303,11 +306,17 @@ def try_from_base64(cls, value) -> FodIdParseResult: The cloud issues a 51Did in the standard alphabet with padding, and a page puts one in a link in the URL-safe alphabet (``-`` and ``_``) without padding. Both are accepted, with or without padding, by - normalising to the standard form before the envelope is read. - - The value may be anything at all, as external data is. ``None`` and - the empty string report :attr:`FodIdParseStatus.MISSING_INPUT`, - anything other than a string reports + normalising to the standard form before the envelope is read. White + space around the value is ignored. A value with white space or any + other character outside the alphabet inside it, or with characters + from both alphabets, reports + :attr:`FodIdParseStatus.INVALID_BASE64`, as + :meth:`to_standard_base64` explains. + + The value may be anything at all, as external data is. ``None``, the + empty string and a string of white space alone report + :attr:`FodIdParseStatus.MISSING_INPUT`, anything other than a string + reports :attr:`FodIdParseStatus.INVALID_INPUT_TYPE`, and every other failure names its reason. The signature is not checked. """ @@ -345,15 +354,33 @@ def from_base64(cls, base64: str) -> "FodId": return cls._from_read_or_raise(_read_base64(base64), "base64") @staticmethod - def to_standard_base64(value: str) -> str: + def to_standard_base64(value: str) -> Optional[str]: """Restores a base64 string in either alphabet to the standard alphabet with padding, which is the form the OWID library decodes. - ``-`` becomes ``+`` and ``_`` becomes ``/``, then ``==`` is added - when the length modulo 4 is 2 and ``=`` when it is 3. A value already - in the standard padded form is returned unchanged. + White space before and after the text is removed first, being every + character with the Unicode White_Space property, so a value read + from a file or a header with a trailing line break reads as the + clean value does. Then ``-`` becomes ``+`` and ``_`` becomes ``/``, + and ``==`` is added when the length modulo 4 is 2 and ``=`` when it + is 3. A value already in the standard padded form is returned + unchanged apart from that trim. + + Answers ``None`` for a text the 51Did specification says to refuse, + being one with white space or any other character outside the + alphabet left inside it, or one holding a character from each + alphabet, such as both ``+`` and ``-``. The issuer writes neither, + so either is a sign of a damaged value, such as a ``+`` a form + decoder turned into a space or a value converted only in part. """ - cleaned = value.strip().replace("-", "+").replace("_", "/") + text = _trim_white_space(value) + if _BASE64_CHARACTERS.fullmatch(text) is None: + return None + # Checked before the mapping below, which would hide a mix. + if (_has_any(text, _STANDARD_ONLY) and + _has_any(text, _URL_SAFE_ONLY)): + return None + cleaned = text.replace("-", "+").replace("_", "/") remainder = len(cleaned) % 4 if remainder == 2: cleaned += "==" @@ -365,9 +392,12 @@ def to_standard_base64(value: str) -> str: def to_base64_url(value: str) -> str: """Converts a base64 string in either alphabet to the URL-safe alphabet without padding, the inverse of :meth:`to_standard_base64`, - so a 51Did can be placed in a URL without further encoding. + so a 51Did can be placed in a URL without further encoding. White + space around the value is removed as :meth:`to_standard_base64` + removes it. """ - return value.strip().replace("+", "-").replace("/", "_").rstrip("=") + return _trim_white_space(value).replace("+", "-").replace( + "/", "_").rstrip("=") @classmethod def from_byte_array(cls, buffer: bytes) -> "FodId": @@ -546,12 +576,49 @@ def signature_status(self, public_pem: str) -> SignatureStatus: return self._owid.signature_status(public_pem) +#: Every character with the Unicode White_Space property, which is what the +#: 51Did specification means by white space. Written out because +#: ``str.strip()`` and ``str.isspace()`` also take U+001C to U+001F, which +#: do not have that property and so are outside the alphabet. +_WHITE_SPACE = ( + "\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000") + +#: The characters a 51Did text may hold once the white space around it is +#: removed, being both alphabets and the padding. Which alphabet the text +#: uses is checked separately, and where the padding sits is left to the OWID +#: reader's decoder. +_BASE64_CHARACTERS = re.compile(r"[A-Za-z0-9+/_=-]*") + +#: The characters for the values 62 and 63 that only the standard alphabet +#: uses. +_STANDARD_ONLY = "+/" + +#: The characters for the values 62 and 63 that only the URL-safe alphabet +#: uses. +_URL_SAFE_ONLY = "-_" + + +def _trim_white_space(value: str) -> str: + """The value without the white space before and after it.""" + return value.strip(_WHITE_SPACE) + + +def _has_any(text: str, characters: str) -> bool: + """Whether the text holds any of the characters.""" + return any(character in text for character in characters) + + def _read_base64(value) -> ParseResult: - """The OWID read of a base64 string in either alphabet. Anything that is - not a string goes to the OWID reader as given, so the reason it reports - (nothing supplied, or a type it cannot read) is the one carried.""" + """The OWID read of a base64 string in either alphabet. A string the + 51Did rules refuse reports invalid base 64, as the OWID reader does for + any other text it cannot decode. Anything that is not a string goes to + the OWID reader as given, so the reason it reports (nothing supplied, or + a type it cannot read) is the one carried.""" if isinstance(value, str): value = FodId.to_standard_base64(value) + if value is None: + return ParseResult(False, None, ParseStatus.INVALID_BASE64) return Owid.parse(value) diff --git a/pipeline/fiftyone_pipeline_did/tests/test_fodid.py b/pipeline/fiftyone_pipeline_did/tests/test_fodid.py index 3b6b445d..dad12a7a 100644 --- a/pipeline/fiftyone_pipeline_did/tests/test_fodid.py +++ b/pipeline/fiftyone_pipeline_did/tests/test_fodid.py @@ -852,6 +852,168 @@ def test_raising_and_non_raising_readers_agree_on_success(self): self.assertEqual(raising.match_key, result.value.match_key) +#: Every character with the Unicode White_Space property, which is what the +#: 51Did specification means by white space. +WHITE_SPACE = ( + "\t\n\x0b\x0c\r \x85\xa0\u1680\u2000\u2001\u2002\u2003\u2004\u2005" + "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000") + +#: Characters ``str.strip()`` removes or that look like white space, but +#: which do not have the White_Space property, so they are outside the +#: alphabet wherever they sit. U+001C to U+001F are the ones Python's own +#: ``str.isspace()`` takes, and U+FEFF and U+200B are zero width +#: characters that cannot be seen in the text. +NOT_WHITE_SPACE = "\x1c\x1d\x1e\x1f\ufeff\u200b" + + +def _forms(standard): + """The standard text, and the URL-safe text with and without its + padding.""" + url_safe = standard.replace("+", "-").replace("/", "_") + return (standard, url_safe, url_safe.rstrip("=")) + + +def _spaced(text, space, count): + """The text with the white space put in at a number of points spread + evenly through it.""" + spaced = text + for point in range(count, 0, -1): + at = len(text) * point // (count + 1) + spaced = spaced[:at] + space + spaced[at:] + return spaced + + +def _show(text): + """The text with each character outside printable ASCII written as its + code, so a failure message shows where it is.""" + return "".join( + c if " " < c <= "~" else "\\u{0:04X}".format(ord(c)) for c in text) + + +class FodIdBase64TextTests(unittest.TestCase): + """How the base64 readers treat the text around and inside a 51Did, as + the identifier layout page of the 51Did specification sets out. White + space before and after the text is ignored, any other character outside + the alphabet is refused, white space inside the text included, and a + text holding a character from each alphabet is refused.""" + + def setUp(self): + self.factory = FodIdTestFactory() + self.standard = self._standard_with_both_specials() + + def _standard_with_both_specials(self): + """The standard text of a real signed 51Did holding both ``+`` and + ``/``, so every form below holds both characters of its alphabet. + The signature differs on every signing, so a few tries find one.""" + for _ in range(100): + standard = self.factory.signed_owid_base64(canonical_payload()) + if "+" in standard and "/" in standard: + return standard + self.fail("No signed envelope contained both + and / in 100 tries.") + + def assert_reads_as_standard(self, text): + result = FodId.try_from_base64(text) + self.assertTrue(result.ok, "{0} for {1}".format( + result.status.value, _show(text))) + self.assertEqual(self.standard, result.value.as_base64()) + self.assertEqual( + self.standard, FodId.from_base64(text).as_base64()) + + def assert_invalid_base64(self, text): + """Both readers refuse the text as invalid base64, and the step + they share gives nothing for it.""" + result = FodId.try_from_base64(text) + self.assertFalse(result.ok, "Parsed {0}".format(_show(text))) + self.assertIsNone(result.value) + self.assertIs(FodIdParseStatus.INVALID_BASE64, result.status, + _show(text)) + with self.assertRaises(OwidError) as raised: + FodId.from_base64(text) + self.assertIn("InvalidBase64", str(raised.exception)) + self.assertIsNone(FodId.to_standard_base64(text), _show(text)) + + def test_the_text_holds_both_specials_in_every_form(self): + standard, url_safe, unpadded = _forms(self.standard) + self.assertTrue("+" in standard and "/" in standard) + self.assertTrue("-" in url_safe and "_" in url_safe) + self.assertTrue("-" in unpadded and "_" in unpadded) + self.assertNotIn("=", unpadded) + + def test_every_form_parses(self): + for text in _forms(self.standard): + with self.subTest(text=text): + self.assert_reads_as_standard(text) + + def test_white_space_around_the_text_parses(self): + for before, after in ((" ", "\n"), ("\t", "\r\n"), + ("\u00a0", "\u00a0"), ("\u3000", "\u3000")): + for text in _forms(self.standard): + with self.subTest(before=_show(before), text=text): + self.assert_reads_as_standard(before + text + after) + + def test_every_white_space_character_around_the_text_parses(self): + self.assertEqual(25, len(WHITE_SPACE)) + for space in WHITE_SPACE: + for text in _forms(self.standard): + with self.subTest(space=_show(space), text=text): + self.assert_reads_as_standard(space + text + space) + + def test_white_space_inside_the_text_is_invalid_base64(self): + for space, count in ((" ", 1), (" ", 2), (" ", 3), ("\t", 4), + ("\r\n", 2), ("\u00a0", 4), ("\u3000", 1)): + for text in _forms(self.standard): + with self.subTest(space=_show(space), count=count, + text=text): + self.assert_invalid_base64(_spaced(text, space, count)) + + def test_every_white_space_character_inside_the_text_is_refused(self): + for space in WHITE_SPACE: + for text in _forms(self.standard): + with self.subTest(space=_show(space), text=text): + self.assert_invalid_base64(_spaced(text, space, 1)) + + def test_characters_that_are_not_white_space_are_refused_anywhere(self): + # Python's own strip() removes U+001C to U+001F, so they are + # checked around the text as well as inside it. + for character in NOT_WHITE_SPACE: + for text in _forms(self.standard): + for spaced in (character + text, text + character, + _spaced(text, character, 1)): + with self.subTest(text=_show(spaced)): + self.assert_invalid_base64(spaced) + + def test_both_alphabets_is_invalid_base64(self): + url_safe = FodId.to_base64_url(self.standard) + for text, from_, to in ((self.standard, "+", "-"), + (self.standard, "/", "_"), + (url_safe, "-", "+"), + (url_safe, "_", "/")): + with self.subTest(from_=from_, to=to): + at = text.index(from_) + self.assert_invalid_base64(text[:at] + to + text[at + 1:]) + + def test_white_space_alone_is_still_missing_input(self): + for text in (" ", "\t\r\n", "\u00a0", "\u3000", WHITE_SPACE): + with self.subTest(text=_show(text)): + result = FodId.try_from_base64(text) + self.assertFalse(result.ok) + self.assertIsNone(result.value) + self.assertIs(FodIdParseStatus.MISSING_INPUT, result.status) + + def test_to_standard_base64_restores_alphabet_and_padding(self): + for value, expected in (("-_", "+/=="), ("+/==", "+/=="), + ("", ""), (" YQ\n", "YQ=="), + ("\u3000-_\u3000", "+/==")): + with self.subTest(value=_show(value)): + self.assertEqual(expected, FodId.to_standard_base64(value)) + + def test_to_standard_base64_gives_none_for_a_text_it_refuses(self): + for value in ("YW Jj", "YW\tJj", "YW\r\nJj", "YW\u00a0Jj", + "YWJj\x1c", "+_", "-/", "+-", "/_"): + with self.subTest(value=_show(value)): + self.assertIsNone(FodId.to_standard_base64(value)) + + class FodIdTermsTests(unittest.TestCase): """The Terms byte, which says which terms document the identifier was created under so that the terms travel with the identifier. The byte is From f55f0d9c67e28f435bdda2007a17f139e841cd8d Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 9 Nov 2026 21:20:38 +0000 Subject: [PATCH 26/26] Use the specification's defaults for the performance profile and Update Matched User Agent (#57) * Leave Update Matched User Agent on by default, as the native code sets it The Device Detection on-premise specification says the option is on by default, so the UserAgents property holds the matched characters of the evidence. The C library sets it on, but the Python engine took update_matched_useragent=False and always passed it on, so UserAgents was a list of empty strings unless a caller turned the option on. The engine now takes None and only sets the option when a value is given, so the native default applies, as the .NET builder does. The docstrings in the engine and both builders say the option is on by default, and the match metrics example's comment no longer says it has to be set. A new test builds a pipeline on the free Lite data file with the option left unset, set on and set off, and skips when that file is not there. * Use the Balanced performance profile by default, as the native code does The Device Detection on-premise specification gives Balanced as the default performance profile, and the C library's default configuration is the Balanced one, but the Python engine defaulted to LowMemory. The engine now defaults to "Balanced", which its profile switch already maps to the native Balanced configuration, and the docstrings in the engine and both builders say so. Balanced loads some collections into memory and caches others, where LowMemory reads everything from the data file, so a caller relying on the default uses more memory. Naming "LowMemory" keeps the previous behaviour. A new test compares the collection settings of the configuration an engine built at its defaults gives the native code with those of each profile, because the engine keeps no name for the profile in use. * Say which profiles the default profile test cannot tell apart MaxPerformance and LowMemory have the same collection settings, as do Balanced and BalancedTemp, so the test's comparison of collection settings cannot separate either pair. The test's docstring now says so. * Say what the matched User-Agent length limit does, as the C header does The max_matched_useragent_length docstrings in the engine and both builders said the option is ignored when update_matched_useragent is false, and named it max_matched_useragents_length. The C library's header (src/dd/config-dd.h) says the limit applies to a single header, a pseudo header and the matched User-Agent, that a longer value is ignored and detection moves on, and that the limit applies whether or not the matched User-Agent is recorded. The docstrings now say the same, give the native default of 500, and spell the option's name correctly. * Drop the cloud builder's false matched User-Agent settings The cloud builder's docstring listed max_matched_useragents_length, misspelled, as ignored when update_matched_useragent is false, which the C header contradicts. The cloud builder reads neither setting, so both entries are removed rather than corrected. The match metrics example comment now says the option is set to show where to turn it off, as the line sets it to True. --- .../devicedetection_pipelinebuilder.py | 22 +++- .../devicedetection_cloud_pipelinebuilder.py | 4 - .../onpremise/match_metrics.py | 5 +- .../devicedetection_onpremise.py | 31 +++-- ...vicedetection_onpremise_pipelinebuilder.py | 22 +++- .../tests/test_performance_profile_default.py | 115 ++++++++++++++++++ .../tests/test_update_matched_useragent.py | 89 ++++++++++++++ 7 files changed, 264 insertions(+), 24 deletions(-) create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py create mode 100644 device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py diff --git a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py index 13a1f624..1a735f87 100644 --- a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py @@ -83,10 +83,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -94,7 +104,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py index 4f59cada..8380803d 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py @@ -81,10 +81,6 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py index dd53ed68..8f364ac8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py @@ -90,8 +90,9 @@ def run(self, data_file, show_descs, logger, output): # https://51degrees.com/documentation/_device_detection__hash.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=run#DeviceDetection_Hash_DataSetProduction_Performance use_predictive_graph = True, use_performance_graph = False, - # We want to show the matching evidence characters as part of this example, so we have to set - # this flag to true. + # This example shows the matched evidence characters, which + # needs this option on. It is on by default, and is set here to + # show where to turn it off. update_matched_useragent = True).add_logger(logger).build() # The pipeline is closed at the end of the with block, which diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index 632751ae..5f3d168c 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -61,8 +61,8 @@ def __init__( polling_interval=30, update_time_maximum_randomisation=10, create_temp_data_copy=True, - update_matched_useragent=False, - performance_profile="LowMemory", + update_matched_useragent=None, + performance_profile="Balanced", reuse_temp_file=False, concurrency=multiprocessing.cpu_count(), update_on_start=False, @@ -90,10 +90,20 @@ def __init__( @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -101,7 +111,9 @@ def __init__( @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int @@ -209,7 +221,10 @@ def __init__( config.setUseTempFile(True) config.setReuseTempFile(reuse_temp_file) - config.setUpdateMatchedUserAgent(update_matched_useragent) + + # Left unset, the native default applies, which is on. + if update_matched_useragent is not None: + config.setUpdateMatchedUserAgent(update_matched_useragent) if allow_unmatched == True or allow_unmatched == False: config.setAllowUnmatched(allow_unmatched) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py index 133b9f2e..28ce1b70 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise_pipelinebuilder.py @@ -80,10 +80,20 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @param licence_keys: licencekeys to use for the data file update service @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type max_matched_useragents_length : int - @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: - @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent + @type max_matched_useragent_length: int + @param max_matched_useragent_length: Limit in bytes on the length of + a single header such as User-Agent, of a pseudo header including the + separators between its values, and of the matched User-Agent. A value + longer than the limit is ignored and detection moves on to the next + header. The limit applies whether or not update_matched_useragent is + on. Left unset, the native default of 500 applies. A larger limit + raises the memory each set of results holds. + @type update_matched_useragent: bool + @param update_matched_useragent: True if the detection should record + the matched characters from the target User-Agent, which the + UserAgents property returns. On by default, as the native code sets + it. Set it to False when UserAgents is not needed, which saves + recording the characters. @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @type difference: int @@ -91,7 +101,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type allow_unmatched : bool @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance + @param performance_profile: options are: LowMemory, MaxPerformance, + Balanced, BalancedTemp, HighPerformance. Balanced by default, as in + the native code. @type reuse_temp_file: bool @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. @type concurrency: int diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py new file mode 100644 index 00000000..a40f04de --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_performance_profile_default.py @@ -0,0 +1,115 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder +from fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule import ConfigHashSwig +from fiftyone_devicedetection_onpremise.devicedetection_onpremise import DeviceDetectionOnPremise +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +# The free Lite data file, where cxx/data/get-data-files.ps1 and +# get-data-files.sh download it and where CI links it in. +DATA_FILE = os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "cxx", + "data", "device-detection", "51Degrees-LiteV4.1.hash")) + +# The collections whose settings the configuration exposes. +COLLECTIONS = ( + "Strings", "Properties", "Values", "Profiles", "Nodes", "ProfileOffsets") + +# Each profile name the engine accepts and the native setter it maps to. +PROFILES = { + "LowMemory": "setLowMemory", + "MaxPerformance": "setMaxPerformance", + "Balanced": "setBalanced", + "BalancedTemp": "setBalancedTemp", + "HighPerformance": "setHighPerformance", +} + + +def collection_settings(config): + """ + Whether each collection is loaded into memory and how many items it + caches, which are what a performance profile sets. + """ + settings = {} + for name in COLLECTIONS: + collection = getattr(config, "get" + name)() + settings[name] = (collection.getLoaded(), collection.getCapacity()) + return settings + + +def profile_settings(setter): + """The collection settings the native code gives a profile.""" + config = ConfigHashSwig() + getattr(config, setter)() + return collection_settings(config) + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free Lite data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class PerformanceProfileDefaultTests(unittest.TestCase): + """ + The engine keeps no name for the profile in use, so these tests compare + the collection settings of the configuration the engine gives the + native code with the settings each profile has there. MaxPerformance + and LowMemory have the same collection settings, and Balanced and + BalancedTemp do too, so neither pair can be told apart this way. + """ + + def _engine_settings(self, **settings): + engine = DeviceDetectionOnPremise( + data_file_path=DATA_FILE, licence_keys="", **settings) + pipeline = PipelineBuilder().add(engine).build() + self.addCleanup(pipeline.close) + self.assertIsNotNone(engine.engine) + return collection_settings(engine.config) + + def test_balanced_and_low_memory_differ(self): + self.assertNotEqual( + profile_settings("setBalanced"), profile_settings("setLowMemory")) + + def test_engine_defaults_use_balanced(self): + self.assertEqual( + profile_settings("setBalanced"), self._engine_settings()) + + def test_builder_defaults_use_balanced(self): + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, + licence_keys="").build() + self.addCleanup(pipeline.close) + engine = pipeline.get_element("device") + self.assertEqual( + profile_settings("setBalanced"), + collection_settings(engine.config)) + + def test_each_named_profile_is_used(self): + for name, setter in PROFILES.items(): + with self.subTest(name): + self.assertEqual( + profile_settings(setter), + self._engine_settings(performance_profile=name)) diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py new file mode 100644 index 00000000..155053c5 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_update_matched_useragent.py @@ -0,0 +1,89 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import os +import unittest + +from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder + +# The free Lite data file, where cxx/data/get-data-files.ps1 and +# get-data-files.sh download it and where CI links it in. +DATA_FILE = os.path.normpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..", "..", "cxx", + "data", "device-detection", "51Degrees-LiteV4.1.hash")) + +MOBILE_UA = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2 like Mac OS X) " + "AppleWebKit/604.4.7 (KHTML, like Gecko) Mobile/15C114") + +# The character the native code writes where the evidence was not matched. +UNMATCHED = "_" + + +@unittest.skipUnless( + os.path.isfile(DATA_FILE), + "The free Lite data file is not at {0}. Download it with " + "cxx/data/get-data-files.ps1 or cxx/data/get-data-files.sh, after " + "'git submodule update --init --recursive'.".format(DATA_FILE)) +class UpdateMatchedUserAgentTests(unittest.TestCase): + """ + The Update Matched User Agent option is on unless it is turned off, so + the UserAgents property holds the characters of the evidence that were + matched. + """ + + def _user_agents(self, **settings): + pipeline = DeviceDetectionOnPremisePipelineBuilder( + data_file_path=DATA_FILE, usage_sharing=False, licence_keys="", + **settings).build() + self.addCleanup(pipeline.close) + flowdata = pipeline.create_flowdata() + self.addCleanup(flowdata.close) + flowdata.evidence.add("header.user-agent", MOBILE_UA) + flowdata.process() + return flowdata.device.useragents.value() + + def assert_matched_from_the_evidence(self, user_agents): + """ + Each matched User-Agent has the evidence's characters where they were + matched, and at least one character was matched. + """ + self.assertGreaterEqual(len(user_agents), 1) + matched = 0 + for user_agent in user_agents: + for index, character in enumerate(user_agent): + if character != UNMATCHED: + self.assertEqual(MOBILE_UA[index], character, user_agent) + matched += 1 + self.assertGreater(matched, 0, user_agents) + + def test_builder_defaults_populate_user_agents(self): + self.assert_matched_from_the_evidence(self._user_agents()) + + def test_turned_on_populates_user_agents(self): + self.assert_matched_from_the_evidence( + self._user_agents(update_matched_useragent=True)) + + def test_turned_off_leaves_user_agents_empty(self): + user_agents = self._user_agents(update_matched_useragent=False) + self.assertGreaterEqual(len(user_agents), 1) + self.assertEqual([""] * len(user_agents), user_agents)