diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py
index 177a42e..7d769b8 100644
--- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py
+++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py
@@ -59,8 +59,8 @@ def test_pipeline_builder_cloud_engine_init(self):
pipeline = DeviceDetectionCloudPipelineBuilder(resource_key = self.resource_key).build()
- self.assertTrue(pipeline.flow_elements[0].datakey == "cloud")
- self.assertTrue(pipeline.flow_elements[1].datakey == "device")
+ self.assertIn("cloud", pipeline.flow_elements_list)
+ self.assertIn("device", pipeline.flow_elements_list)
def test_properties_cloud(self):
"""!
@@ -69,7 +69,7 @@ def test_properties_cloud(self):
pipeline = DeviceDetectionCloudPipelineBuilder(resource_key = self.resource_key).build()
- properties = pipeline.flow_elements[1].get_properties()
+ properties = pipeline.get_element("device").get_properties()
self.assertTrue(len(properties.keys()) > 0)
diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py
index 4742eed..2f7a633 100644
--- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py
+++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/gettingstarted_console.py
@@ -174,8 +174,8 @@ def main(argv):
# somewhere in the project space, or you may specify another file as
# a command line parameter.
#
- # Note that the Lite data file is only used for illustration, and has
- # limited accuracy and capabilities. The example shows the properties
+ # Note that the Lite data file is only used for illustration, and has a small
+ # number of Properties. The example shows the properties
# only the Enterprise data file has as needing it. Contact us for a copy
# of the Enterprise data file:
# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-gettingstarted_console.py&utm_term=main
diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py
index dd53ed6..a52b55f 100644
--- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py
+++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/match_metrics.py
@@ -188,8 +188,8 @@ def main(argv):
# somewhere in the project space, or you may specify another file as
# a command line parameter.
#
- # Note that the Lite data file is only used for illustration, and has
- # limited accuracy and capabilities.
+ # Note that the Lite data file is only used for illustration, and has a small
+ # number of Properties.
# Contact us for a copy of the Enterprise data file:
# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-match_metrics.py&utm_term=main
data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME)
diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py
index e3141e0..f6334c3 100644
--- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py
+++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/metadata_console.py
@@ -126,8 +126,8 @@ def main(argv):
# somewhere in the project space, or you may specify another file as
# a command line parameter.
#
- # Note that the Lite data file is only used for illustration, and has
- # limited accuracy and capabilities.
+ # Note that the Lite data file is only used for illustration, and has a small
+ # number of Properties.
# Contact us for a copy of the Enterprise data file:
# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-metadata_console.py&utm_term=main
data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME)
diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py
index c4b437f..9dcb71e 100644
--- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py
+++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/offlineprocessing.py
@@ -222,8 +222,8 @@ def main(argv):
# somewhere in the project space, or you may specify another file as
# a command line parameter.
#
- # Note that the Lite data file is only used for illustration, and has
- # limited accuracy and capabilities.
+ # Note that the Lite data file is only used for illustration, and has a small
+ # number of Properties.
# Contact us for a copy of the Enterprise data file:
# https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-onpremise-offlineprocessing.py&utm_term=main
data_file = argv[0] if len(argv) > 0 else ExampleUtils.find_file(LITE_DATAFILE_NAME)
diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py
index 632751a..4f0c180 100644
--- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py
+++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py
@@ -433,8 +433,9 @@ def process_internal(self, flow_data):
evidence = EvidenceDeviceDetectionSwig()
+ # The native evidence takes text, and the Sequence Element adds a number.
for key, value in flow_data.evidence.get_all().items():
- evidence.__setitem__(key, value)
+ evidence.__setitem__(key, str(value))
result = self.engine.process(evidence)
diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py
index e1ac994..47cfb44 100644
--- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py
+++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_automatic_data_file_updates.py
@@ -73,7 +73,7 @@ def test_on_premise_data_update_url(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertFalse(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -92,7 +92,7 @@ def test_on_premise_data_old_property_data_file_update_base_url(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertFalse(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -114,7 +114,7 @@ def test_on_premise_data_update_url_config(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertFalse(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -136,7 +136,7 @@ def test_on_premise_data_config_old_property_data_file_update_base_url(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertFalse(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -155,7 +155,7 @@ def test_on_premise_data_update_verify_md5(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertTrue(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -174,7 +174,7 @@ def test_on_premise_data_update_verify_md5_old_property(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertTrue(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -196,7 +196,7 @@ def test_on_premise_data_update_verify_md5_config(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertTrue(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -218,7 +218,7 @@ def test_on_premise_data_update_verify_md5_config_old_property(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertTrue(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
@@ -238,7 +238,7 @@ def test_on_premise_data_update_use_url_formatter(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self._assertNotSignedUrl(flow_element_data_file.get_update_url())
@@ -260,7 +260,7 @@ def test_on_premise_data_update_use_url_formatter_config(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self._assertNotSignedUrl(flow_element_data_file.get_update_url())
@@ -278,7 +278,7 @@ def test_on_premise_data_update_status_rate_limit(self):
with pipeline.create_flowdata() as flowdata:
flowdata.process()
- flow_element_data_file = pipeline.flow_elements[0].data_file
+ flow_element_data_file = pipeline.get_element("device").data_file
self.assertIsInstance(flow_element_data_file, DeviceDetectionDataFile)
self.assertTrue(flow_element_data_file.verify_md5)
self._assertSignedUrl(flow_element_data_file.get_update_url())
diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py
index 11ed21f..af63494 100644
--- a/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py
+++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py
@@ -63,7 +63,7 @@ def test_pipeline_builder_shareusage_init(self):
data_file_path=data_file, licence_keys="").build()
self.addCleanup(pipeline.close)
- self.assertTrue(pipeline.flow_elements[0].datakey == "shareusage")
+ self.assertIn("shareusage", pipeline.flow_elements_list)
def test_on_premise_engine_datafile(self):
"""!
@@ -75,7 +75,7 @@ def test_on_premise_engine_datafile(self):
self.addCleanup(pipeline.close)
self.assertTrue(isinstance(
- pipeline.flow_elements[0].data_file, DeviceDetectionDataFile))
+ pipeline.get_element("device").data_file, DeviceDetectionDataFile))
def test_properties_onpremise(self):
"""!
@@ -87,7 +87,7 @@ def test_properties_onpremise(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- properties = pipeline.flow_elements[0].get_properties()
+ properties = pipeline.get_element("device").get_properties()
self.assertTrue(len(properties.keys()) > 0)
@@ -100,7 +100,7 @@ def test_evidencekey_filter_onpremise(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter(
+ evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter(
)
self.assertFalse(evidence_key_filter.filter_evidence_key("test.test"))
@@ -310,7 +310,7 @@ def test_evidencekey_filter_contains_user_agent(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter(
+ evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter(
)
self.assertTrue(
@@ -325,7 +325,7 @@ def test_evidencekey_filter_contains_device_stock_ua(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter(
+ evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter(
)
self.assertTrue(evidence_key_filter.filter_evidence_key(
@@ -340,7 +340,7 @@ def test_evidencekey_filter_contains_query_params(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter(
+ evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter(
)
self.assertTrue(evidence_key_filter.filter_evidence_key(
@@ -357,7 +357,7 @@ def test_evidencekey_filter_case_insensitive_keys(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter(
+ evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter(
)
self.assertTrue(
@@ -378,7 +378,7 @@ def test_evidencekey_filter_overrides(self):
data_file_path=data_file, usage_sharing=False, licence_keys="").build()
self.addCleanup(pipeline.close)
- evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter(
+ evidence_key_filter = pipeline.get_element("device").get_evidence_key_filter(
)
self.assertTrue(evidence_key_filter.filter_evidence_key(
diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py
index 9f15784..e7f9a01 100644
--- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py
+++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/ipintelligence_onpremise.py
@@ -400,8 +400,9 @@ def process_internal(self, flow_data):
evidence = EvidenceIpiSwig()
+ # The native evidence takes text, and the Sequence Element adds a number.
for key, value in flow_data.evidence.get_all().items():
- evidence.__setitem__(key, value)
+ evidence.__setitem__(key, str(value))
result = self.engine.process(evidence)
diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py
index bdb85c5..a97e235 100644
--- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py
+++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/javascriptbuilder.py
@@ -351,7 +351,13 @@ def process_internal(self, flowdata):
for key, value in self.settings.items():
variables[key] = value
- variables["_jsonObject"] = json.dumps(flowdata.jsonbundler.json)
+ # The JSON is written into a script element, so the characters that
+ # could end the element or the script are written as escapes.
+ variables["_jsonObject"] = (
+ json.dumps(flowdata.jsonbundler.json)
+ .replace("<", "\\u003c")
+ .replace(">", "\\u003e")
+ .replace("&", "\\u0026"))
# Generate URL and autoUpdate params
diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py
index 9f2a8ca..b11f6fe 100644
--- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py
+++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py
@@ -70,7 +70,6 @@ def process_internal(self, flowdata):
for flow_element in flowdata.pipeline.flow_elements:
if (flow_element.datakey == "jsonbundler" or
- flow_element.datakey == "sequence" or
flow_element.datakey == "javascriptbuilder" or
flow_element.datakey == "set-headers"):
continue
diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py
index c3fa52e..33e7829 100644
--- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py
+++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py
@@ -42,12 +42,22 @@ def __init__(self, settings= {}):
Pipeline Builder constructor.
@type settings: dictionary
@param settings : settings for the pipeline builder including:
- `add_javascript_builder (Bool)` - Whether to add the JavaScriptBuilder to the pipeline
- (default true)
+ `add_javascript_builder (Bool)` - Whether to add the JavaScriptBuilder, with the
+ JSONBundler and SequenceElement it works with, to the pipeline (default true).
+ Set to False to add none of the three.
+ `add_sequence_element (Bool)` - Whether to add the SequenceElement, as the first
+ element of the pipeline (default true)
+ `add_json_builder_element (Bool)` - Whether to add the JSONBundler (default true).
+ The JavaScriptBuilder reads its output, so a pipeline that keeps the
+ JavaScriptBuilder needs a JSONBundler, added here or by the caller.
+ `add_javascript_builder_element (Bool)` - Whether to add the JavaScriptBuilder
+ alone (default true)
`javascript_builder_settings (dict)` - Settings for the JavaScriptBuilder engine
@rtype: PipelineBuilder
`use_setheader_properties (Bool)` - Whether to add the SetHeaderElement to the pipeline
(default true)
+ `add_set_header_element (Bool)` - The same switch as use_setheader_properties.
+ The SetHeaderElement is added only when both are true (default true)
`suppress_process_exceptions (Bool)` - If False (the default) the first error thrown
while processing is re-raised from flow_data.process(); if True, errors are stored on
flow_data.errors and process() returns normally. Recommended True for web apps so a
@@ -67,6 +77,14 @@ def __init__(self, settings= {}):
else:
self.add_javaScriptbuilder = True
+ self.add_sequence_element = settings.get("add_sequence_element", True)
+ self.add_json_builder_element = settings.get(
+ "add_json_builder_element", True)
+ self.add_javascript_builder_element = settings.get(
+ "add_javascript_builder_element", True)
+ self.add_set_header_element = settings.get(
+ "add_set_header_element", True)
+
if "javascript_builder_settings" in settings:
self.javascriptbuilder_settings = settings["javascript_builder_settings"]
@@ -90,10 +108,25 @@ def __init__(self, settings= {}):
else:
self.auto_close_elements = True
+ def get_sequence_elements(self):
+ """
+ The SequenceElement, to be the first element of the pipeline, unless
+ add_javascript_builder or add_sequence_element is false
+ @rtype: list
+ @return: Returns a list of FlowElements
+ """
+
+ if self.add_javaScriptbuilder and self.add_sequence_element:
+ return [SequenceElement()]
+
+ return []
+
def get_javascript_elements(self):
"""
- Adds the JavaScriptBuilder, JSONBundler and SequenceElement to the pipeline if
- If add_javascript_builder is set to true (the default)
+ The JSONBundler and the JavaScriptBuilder, to be added after the other
+ elements. Neither is added if add_javascript_builder is false, and
+ each has its own switch (add_json_builder_element and
+ add_javascript_builder_element).
@rtype: list
@return: Returns a list of FlowElements
"""
@@ -102,27 +135,28 @@ def get_javascript_elements(self):
if (self.add_javaScriptbuilder):
- flow_elements.append(SequenceElement())
- flow_elements.append(JSONBundlerElement())
-
- if (hasattr(self, "javascriptbuilder_settings")):
- flow_elements.append(JavascriptBuilderElement(self.javascriptbuilder_settings))
- else:
- flow_elements.append(JavascriptBuilderElement())
+ if self.add_json_builder_element:
+ flow_elements.append(JSONBundlerElement())
+
+ if self.add_javascript_builder_element:
+ if (hasattr(self, "javascriptbuilder_settings")):
+ flow_elements.append(JavascriptBuilderElement(self.javascriptbuilder_settings))
+ else:
+ flow_elements.append(JavascriptBuilderElement())
return flow_elements
def get_setheader_elements(self):
"""
Adds the SetHeaderElement to the pipeline if
- If use_setheader_properties is set to true (the default)
+ If use_setheader_properties and add_set_header_element are true (the default)
@rtype: list
@return: Returns a list of FlowElements
"""
flow_elements = []
- if (self.use_setheader_properties):
+ if self.use_setheader_properties and self.add_set_header_element:
flow_elements.append(SetHeaderElement())
return flow_elements
@@ -153,10 +187,13 @@ def build(self):
"""
- self.flow_elements.extend(self.get_javascript_elements())
- self.flow_elements.extend(self.get_setheader_elements())
+ flow_elements = (
+ self.get_sequence_elements() +
+ self.flow_elements +
+ self.get_javascript_elements() +
+ self.get_setheader_elements())
- return Pipeline(self.flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements)
+ return Pipeline(flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements)
def add_logger(self, logger):
"""
diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py
index c405623..c7cc251 100644
--- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py
+++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/sequenceelement.py
@@ -24,8 +24,13 @@
from .flowelement import FlowElement
from .evidence_keyfilter import EvidenceKeyFilter
from .javascriptbuilder import parse_sequence
+from .elementdata_dictionary import ElementDataDictionary
+import re
import uuid
+# Text that is a sequence, in ASCII decimal digits and nothing else.
+DIGITS_PATTERN = re.compile(r"[0-9]+")
+
class SequenceElementEvidenceKeyFilter(EvidenceKeyFilter):
def filter(self, key):
@@ -52,6 +57,9 @@ def __init__(self):
super(SequenceElement, self).__init__()
self.datakey = "sequence"
+ self.properties = {
+ "session-id": {"type": "string"},
+ "sequence": {"type": "int"}}
self.exclude_from_messages = True
@@ -66,30 +74,30 @@ def get_evidence_key_filter(self):
def process_internal(self, flowdata):
"""!
- The SequenceElement stores session data for requests for JavaScript
+ The SequenceElement stores session data for requests for JavaScript,
+ in its Element Data (session-id and sequence) and in Evidence
@type flowdata: FlowData
@param flowdata: The FlowData
"""
- if flowdata.evidence.get("query.session-id"):
-
- # Get current sequence number. A value that is absent or is not
- # a positive 32 bit integer is treated as no sequence at all, so
- # the request is sequence 1 rather than failing. The JavaScript
- # builder renders any value it cannot use as 1 in the same way.
-
- sequence = parse_sequence(flowdata.evidence.get("query.sequence"))
-
- flowdata.evidence.add(
- "query.sequence",
- 1 if sequence is None else sequence + 1)
-
- else:
- flowdata.evidence.add(
- "query.session-id",
- str(uuid.uuid4())
- )
-
- flowdata.evidence.add("query.sequence", 1)
+ # The session id and the sequence are read independently. A session
+ # id that is not a non-empty string is absent, and gets a new id. A
+ # sequence that is not a whole number from 1 to 2147483647, written
+ # in decimal digits if it is text, is absent and gives 1.
+ session_id = flowdata.evidence.get("query.session-id")
+ if not isinstance(session_id, str) or session_id == "":
+ session_id = str(uuid.uuid4())
+
+ sequence = flowdata.evidence.get("query.sequence")
+ if isinstance(sequence, str) and not DIGITS_PATTERN.fullmatch(sequence):
+ sequence = None
+ sequence = parse_sequence(sequence)
+ sequence = 1 if sequence is None else sequence + 1
+
+ flowdata.evidence.add("query.session-id", session_id)
+ flowdata.evidence.add("query.sequence", sequence)
+
+ flowdata.set_element_data(ElementDataDictionary(
+ self, {"session-id": session_id, "sequence": sequence}))
diff --git a/pipeline/fiftyone_pipeline_core/tests/test_core.py b/pipeline/fiftyone_pipeline_core/tests/test_core.py
index 21407b3..9f0d88b 100644
--- a/pipeline/fiftyone_pipeline_core/tests/test_core.py
+++ b/pipeline/fiftyone_pipeline_core/tests/test_core.py
@@ -86,8 +86,9 @@ def testGet(self):
def testGetWhere(self):
testPipeline = TestPipeline()
+ # The example element's integer and the Sequence Element's sequence.
getValue = len(testPipeline.flowdata.get_where("type", "int"))
- self.assertTrue(getValue == 1)
+ self.assertTrue(getValue == 2)
def testGetFromElement(self):
diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py
index a2d80b0..03f4e68 100644
--- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py
+++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder.py
@@ -229,7 +229,13 @@ def testJSONBundler(self):
}
}
- self.assertEqual(FlowData.jsonbundler.json, expected)
+ # The Sequence Element has an entry too. Its session id is new on
+ # every request.
+ actual = dict(FlowData.jsonbundler.json)
+ sequence = actual.pop('sequence')
+ self.assertTrue(sequence['session-id'])
+ self.assertEqual(1, sequence['sequence'])
+ self.assertEqual(actual, expected)
def testJavaScriptBuilder_Minify(self):
diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py
index 2e42c30..3c7093b 100644
--- a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py
+++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_sequence.py
@@ -211,7 +211,9 @@ def test_longest_session_id_is_rendered(self):
@parameterized.expand([
[name + "_" + ("with" if element else "without")
- + "_sequence_element", value, check_text, element]
+ + "_sequence_element", value,
+ # The JSON of the Sequence Element's data holds the long id as it is.
+ check_text and not (element and name == "too_long"), element]
for name, value, check_text in [
["quote", 'a"b', True],
["backslash", "a\\b", True],
@@ -231,7 +233,8 @@ def test_unsafe_session_id_is_rendered_empty(
one that is not 1 to 64 ASCII letters, digits and hyphens is rendered
as an empty string, as the pipeline specification says. The
SequenceElement keeps a session id it is given, so the check is the
- builder's in both pipelines."""
+ builder's in both pipelines. The JSON in the script cannot be ended
+ early by the id either."""
script = _render(
{"query.session-id": value},
diff --git a/pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py b/pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py
new file mode 100644
index 0000000..4be3fcd
--- /dev/null
+++ b/pipeline/fiftyone_pipeline_core/tests/test_web_integration_elements.py
@@ -0,0 +1,190 @@
+
+"""
+The elements the web integration adds (Sequence, JSON Builder, JavaScript
+Builder and Set Headers) each have a switch that defaults to on, and the
+Sequence Element comes first, as the web integration specification says.
+"""
+
+import unittest
+
+from fiftyone_pipeline_core.flowelement import FlowElement
+from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder
+
+
+class UserElement(FlowElement):
+
+ def __init__(self):
+ super(UserElement, self).__init__()
+ self.datakey = "user"
+
+ def process_internal(self, flowdata):
+ pass
+
+
+def datakeys(settings=None):
+ builder = PipelineBuilder(settings or {})
+ builder.add(UserElement())
+ pipeline = builder.build()
+ return [element.datakey for element in pipeline.flow_elements]
+
+
+class WebIntegrationElementSwitchTests(unittest.TestCase):
+
+ def testAllElementsAreAddedByDefaultWithSequenceFirst(self):
+ self.assertEqual(
+ ["sequence", "user", "jsonbundler", "javascriptbuilder",
+ "set-headers"],
+ datakeys())
+
+ def testSequenceSwitch(self):
+ self.assertEqual(
+ ["user", "jsonbundler", "javascriptbuilder", "set-headers"],
+ datakeys({"add_sequence_element": False}))
+
+ def testJsonBuilderSwitch(self):
+ self.assertEqual(
+ ["sequence", "user", "javascriptbuilder", "set-headers"],
+ datakeys({"add_json_builder_element": False}))
+
+ def testJavaScriptBuilderSwitch(self):
+ self.assertEqual(
+ ["sequence", "user", "jsonbundler", "set-headers"],
+ datakeys({"add_javascript_builder_element": False}))
+
+ def testSetHeadersSwitch(self):
+ self.assertEqual(
+ ["sequence", "user", "jsonbundler", "javascriptbuilder"],
+ datakeys({"add_set_header_element": False}))
+
+ def testExistingJavaScriptSwitchStillDropsThreeElements(self):
+ self.assertEqual(
+ ["user", "set-headers"],
+ datakeys({"add_javascript_builder": False}))
+
+ def testExistingSetHeaderSwitchStillDropsSetHeaders(self):
+ self.assertEqual(
+ ["sequence", "user", "jsonbundler", "javascriptbuilder"],
+ datakeys({"use_setheader_properties": False}))
+
+ def testEverySwitchOff(self):
+ self.assertEqual(
+ ["user"],
+ datakeys({
+ "add_sequence_element": False,
+ "add_json_builder_element": False,
+ "add_javascript_builder_element": False,
+ "add_set_header_element": False}))
+
+
+# The Sequence Element writes Element Data as well as Evidence.
+class SequenceElementDataTests(unittest.TestCase):
+
+ def process(self, evidence):
+ pipeline = PipelineBuilder({}).build()
+ flowdata = pipeline.create_flowdata()
+ for key, value in evidence.items():
+ flowdata.evidence.add(key, value)
+ flowdata.process()
+ return flowdata
+
+ def testFirstRequestHasNewSessionAndSequenceOne(self):
+ flowdata = self.process({})
+ session_id = flowdata.get("sequence").get("session-id")
+ self.assertTrue(session_id)
+ self.assertEqual(1, flowdata.get("sequence").get("sequence"))
+ self.assertEqual(session_id, flowdata.evidence.get("query.session-id"))
+ self.assertEqual(1, flowdata.evidence.get("query.sequence"))
+
+ def testCallbackKeepsSessionAndAddsOne(self):
+ flowdata = self.process(
+ {"query.session-id": "abc-123", "query.sequence": "3"})
+ self.assertEqual("abc-123", flowdata.get("sequence").get("session-id"))
+ self.assertEqual(4, flowdata.get("sequence").get("sequence"))
+ self.assertEqual("abc-123", flowdata.evidence.get("query.session-id"))
+ self.assertEqual(4, flowdata.evidence.get("query.sequence"))
+
+ # The JSON Builder specification requires a top-level entry for each
+ # Element Data, excluding only Set Headers, the Cloud Request Engine and
+ # Usage Sharing.
+ def testSequenceHasAnEntryInTheJsonSentToBrowsers(self):
+ flowdata = self.process({})
+ entry = flowdata.jsonbundler.json["sequence"]
+ self.assertEqual(
+ flowdata.get("sequence").get("session-id"), entry["session-id"])
+ self.assertEqual(1, entry["sequence"])
+
+ # The JSON is written into the script, so a session id from the query
+ # string must not end the script.
+ def testSessionIdIsKeptButCannotEndTheScript(self):
+ flowdata = self.process({"query.session-id": ""})
+ self.assertEqual("", flowdata.get("sequence").get("session-id"))
+ self.assertEqual(
+ "", flowdata.jsonbundler.json["sequence"]["session-id"])
+ self.assertNotIn("", flowdata.javascriptbuilder.javascript)
+ self.assertIn(
+ "\\u003c/script\\u003e", flowdata.javascriptbuilder.javascript)
+
+ def testSequencePropertiesAreDescribed(self):
+ element = PipelineBuilder({}).build().flow_elements[0]
+ self.assertEqual("sequence", element.datakey)
+ self.assertEqual(
+ {"session-id": {"type": "string"}, "sequence": {"type": "int"}},
+ element.get_properties())
+
+
+# The session id and the sequence are read independently, as the Process
+# section of the Sequence Element specification says, and the Evidence and the
+# Element Data are written from the same values.
+class SequenceElementProcessTests(unittest.TestCase):
+
+ def process(self, evidence):
+ pipeline = PipelineBuilder({}).build()
+ flowdata = pipeline.create_flowdata()
+ for key, value in evidence.items():
+ flowdata.evidence.add(key, value)
+ flowdata.process()
+ data = flowdata.get("sequence")
+ session_id = data.get("session-id")
+ sequence = data.get("sequence")
+ self.assertEqual(session_id, flowdata.evidence.get("query.session-id"))
+ self.assertEqual(sequence, flowdata.evidence.get("query.sequence"))
+ return session_id, sequence
+
+ def testSequenceWithNoSessionIdCountsUpWithANewId(self):
+ session_id, sequence = self.process({"query.sequence": "3"})
+ self.assertTrue(session_id)
+ self.assertEqual(4, sequence)
+
+ def testSessionIdWithNoSequenceGivesOne(self):
+ self.assertEqual(("abc", 1), self.process({"query.session-id": "abc"}))
+
+ def testIntegerSequenceCountsUp(self):
+ self.assertEqual(
+ ("abc", 8),
+ self.process({"query.session-id": "abc", "query.sequence": 7}))
+
+ def testSessionIdThatIsNotANonEmptyStringIsAbsent(self):
+ for value in (123, "", None, ["abc"]):
+ session_id, sequence = self.process(
+ {"query.session-id": value, "query.sequence": "2"})
+ self.assertIsInstance(session_id, str)
+ self.assertTrue(session_id)
+ self.assertEqual(3, sequence)
+
+ def testSessionIdOfZeroIsKept(self):
+ self.assertEqual(
+ ("0", 3),
+ self.process({"query.session-id": "0", "query.sequence": "2"}))
+
+ def testSequenceThatIsNotWholeDigitsIsAbsent(self):
+ for value in ("abc", "", " 5 ", "+5", "-5", "5.0", "0", True, 0,
+ 2 ** 31, "2147483648", "٣", None):
+ self.assertEqual(
+ ("abc", 1),
+ self.process(
+ {"query.session-id": "abc", "query.sequence": value}),
+ repr(value))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py
index 4f1bea4..b354cd8 100644
--- a/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py
+++ b/pipeline/fiftyone_pipeline_engines_fiftyone/tests/test_usage.py
@@ -293,7 +293,9 @@ def test_session_and_sequence(self):
self.waitForUsageThreads()
self.assertEqual(len(app.received), 1)
self.assertIn("1", app.received[0])
- self.assertIn("1", app.received[0])
+ # The Sequence Element is the first element, so it has added one to the
+ # sequence by the time usage sharing reads it.
+ self.assertIn("2", app.received[0])
def test_invalid_char(self):
"""!