Replace all uses of SON with dict

docs/changelog.rst

@@ -19,22 +19,23 @@ Changes in 1.0.0
     - Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions)
 - Add a warning that ``mongoengine.org`` is no longer controlled by the MongoEngine
   project and appears to be an expired domain takeover.
-- Fix querying GenericReferenceField with __in operator #2886
-- Fix Document.compare_indexes() not working correctly for text indexes on multiple fields #2612
+- Bug Fix - Fix querying GenericReferenceField with __in operator #2886
+- Bug Fix - Fix Document.compare_indexes() not working correctly for text indexes on multiple fields #2612
 - BREAKING CHANGE: wrap _document_registry (normally not used by end users) with _DocumentRegistry which acts as a singleton to access the registry
 - Log a warning in case users creates multiple Document classes with the same name as it can lead to unexpected behavior #1778
-- Fix use of $geoNear or $collStats in aggregate #2493
+- BugFix - Fix use of $geoNear or $collStats in aggregate #2493
 - BREAKING CHANGE: Further to the deprecation warning, remove ability to use an unpacked list to `Queryset.aggregate(*pipeline)`, a plain list must be provided instead `Queryset.aggregate(pipeline)`, as it's closer to pymongo interface
 - BREAKING CHANGE: Further to the deprecation warning, remove `full_response` from `QuerySet.modify` as it wasn't supported with Pymongo 3+
 - BREAKING CHANGE: Remove deprecated ``QuerySet.snapshot``, which had no effect with PyMongo 3+. Remove calls to ``.snapshot(...)``; there is no direct replacement.
 - BREAKING CHANGE: Remove the deprecated ``Q.empty`` and ``QNode.empty`` properties. Use ``not query`` instead (or ``bool(query)`` for the inverse). #2919
 - Fixed stacklevel of many warnings (to point places emitting the warning more accurately)
 - Add support for collation/hint/comment to delete/update and aggregate #2842
 - BREAKING CHANGE: Remove LongField as it's equivalent to IntField since we drop support to Python2 long time ago (User should simply switch to IntField) #2309
+- Replace MongoEngine-created ``bson.SON`` objects with built-in dictionaries, SON providing no advantages since Python 3.7 as native dict preserved insertion order. #2898
 - BREAKING CHANGE: The obsolete ``slaves`` and ``is_slave`` connection options were silently ignored since 2014 and will now raise ``ConnectionFailure`` if provided #2920.
 - BugFix - Calling .clear on a ListField wasn't being marked as changed (and flushed to db upon .save()) #2858
 - Improve error message in case a document assigned to a ReferenceField wasn't saved yet #1955
-- Fix inc/dec atomic updates rejecting deltas outside a field's min_value/max_value #2339
+- BugFix - Fix inc/dec atomic updates rejecting deltas outside a field's min_value/max_value #2339
 - BugFix - Take `where()` into account when using `.modify()`, as in MyDocument.objects().where("this[field] >= this[otherfield]").modify(field='new') #2044
 
 Changes in 0.29.3

mongoengine/base/document.py

@@ -103,7 +103,7 @@ def __init__(self, *args, **values):
         else:
             self._data = {}
 
-        self._dynamic_fields = SON()
+        self._dynamic_fields = {}
 
         # Assign default values for fields
         # not set in the constructor
@@ -220,10 +220,11 @@ def __getstate__(self):
             if hasattr(self, k):
                 data[k] = getattr(self, k)
         data["_data"] = self.to_mongo()
+        data["_data_is_mongo"] = True
         return data
 
     def __setstate__(self, data):
-        if isinstance(data["_data"], SON):
+        if data.pop("_data_is_mongo", False) or isinstance(data["_data"], SON):
             data["_data"] = self.__class__._from_son(data["_data"])._data
         for k in (
             "_changed_fields",
@@ -241,7 +242,7 @@ def __setstate__(self, data):
                 _super_fields_ordered = type(self)._fields_ordered
                 self._fields_ordered = _super_fields_ordered
 
-        dynamic_fields = data.get("_dynamic_fields") or SON()
+        dynamic_fields = data.get("_dynamic_fields") or {}
         for k in dynamic_fields.keys():
             setattr(self, k, data["_data"].get(k))
 
@@ -331,11 +332,11 @@ def get_text_score(self):
 
     def to_mongo(self, use_db_field=True, fields=None):
         """
-        Return as SON data ready for use with MongoDB.
+        Return as a dictionary ready for use with MongoDB.
         """
         fields = fields or []
 
-        data = SON()
+        data = {}
         data["_id"] = None
         data["_cls"] = self._class_name
 

mongoengine/base/fields.py

@@ -4,7 +4,7 @@
 import weakref
 
 import pymongo
-from bson import SON, DBRef, ObjectId
+from bson import DBRef, ObjectId
 
 from mongoengine.base.common import UPDATE_OPERATORS
 from mongoengine.base.datastructures import (
@@ -748,4 +748,4 @@ def _validate_multipolygon(self, value):
     def to_mongo(self, value):
         if isinstance(value, dict):
             return value
-        return SON([("type", self._type), ("coordinates", value)])
+        return {"type": self._type, "coordinates": value}

mongoengine/dereference.py

@@ -1,4 +1,4 @@
-from bson import SON, DBRef
+from bson import DBRef
 
 from mongoengine.base import (
     BaseDict,
@@ -130,7 +130,7 @@ def _find_references(self, items, depth=0):
                         continue
                     elif isinstance(v, DBRef):
                         reference_map.setdefault(field.document_type, set()).add(v.id)
-                    elif isinstance(v, (dict, SON)) and "_ref" in v:
+                    elif isinstance(v, dict) and "_ref" in v:
                         reference_map.setdefault(
                             _DocumentRegistry.get(v["_cls"]), set()
                         ).add(v["_ref"].id)
@@ -150,7 +150,7 @@ def _find_references(self, items, depth=0):
                 continue
             elif isinstance(item, DBRef):
                 reference_map.setdefault(item.collection, set()).add(item.id)
-            elif isinstance(item, (dict, SON)) and "_ref" in item:
+            elif isinstance(item, dict) and "_ref" in item:
                 reference_map.setdefault(
                     _DocumentRegistry.get(item["_cls"]), set()
                 ).add(item["_ref"].id)
@@ -229,7 +229,7 @@ def _attach_objects(self, items, depth=0, instance=None, name=None):
                 else:
                     return BaseList(items, instance, name)
 
-        if isinstance(items, (dict, SON)):
+        if isinstance(items, dict):
             if "_ref" in items:
                 return self.object_map.get(
                     (items["_ref"].collection, items["_ref"].id), items
@@ -272,7 +272,7 @@ def _attach_objects(self, items, depth=0, instance=None, name=None):
                         data[k]._data[field_name] = self.object_map.get(
                             (v.collection, v.id), v
                         )
-                    elif isinstance(v, (dict, SON)) and "_ref" in v:
+                    elif isinstance(v, dict) and "_ref" in v:
                         data[k]._data[field_name] = self.object_map.get(
                             (v["_ref"].collection, v["_ref"].id), v
                         )

mongoengine/document.py

@@ -116,7 +116,7 @@ def __setstate__(self, state):
     def to_mongo(self, *args, **kwargs):
         data = super().to_mongo(*args, **kwargs)
 
-        # remove _id from the SON if it's in it and it's None
+        # remove _id from the data if it's in it and it's None
         if "_id" in data and data["_id"] is None:
             del data["_id"]
 
@@ -303,7 +303,7 @@ def to_mongo(self, *args, **kwargs):
         data = super().to_mongo(*args, **kwargs)
 
         # If '_id' is None, try and set it from self._data. If that
-        # doesn't exist either, remove '_id' from the SON completely.
+        # doesn't exist either, remove '_id' from the data completely.
         if data["_id"] is None:
             if self._data.get("id") is None:
                 del data["_id"]

mongoengine/fields.py

@@ -12,7 +12,7 @@
 
 import gridfs
 import pymongo
-from bson import SON, Binary, DBRef, ObjectId
+from bson import Binary, DBRef, ObjectId
 from bson.decimal128 import Decimal128, create_decimal128_context
 from pymongo import ReturnDocument
 
@@ -824,7 +824,7 @@ def to_python(self, value):
         return value
 
     def validate(self, value, clean=True):
-        if self.choices and isinstance(value, SON):
+        if self.choices and isinstance(value, dict):
             for choice in self.choices:
                 if value["_cls"] == choice._class_name:
                     return True
@@ -1385,14 +1385,14 @@ def to_mongo(self, document, use_db_field=True, fields=None):
         else:
             self.error("Only accept a document object")
 
-        value = SON((("_id", id_field.to_mongo(id_)),))
+        value = {"_id": id_field.to_mongo(id_)}
 
         if fields:
             new_fields = [f for f in self.fields if f in fields]
         else:
             new_fields = self.fields
 
-        value.update(dict(document.to_mongo(use_db_field, fields=new_fields)))
+        value.update(document.to_mongo(use_db_field, fields=new_fields))
         return value
 
     def prepare_query_value(self, op, value):
@@ -1506,10 +1506,10 @@ def __get__(self, instance, owner):
         return super().__get__(instance, owner)
 
     def validate(self, value):
-        if not isinstance(value, (Document, DBRef, dict, SON)):
+        if not isinstance(value, (Document, DBRef, dict)):
             self.error("GenericReferences can only contain documents")
 
-        if isinstance(value, (dict, SON)):
+        if isinstance(value, dict):
             if "_ref" not in value or "_cls" not in value:
                 self.error("GenericReferences can only contain documents")
 
@@ -1521,7 +1521,7 @@ def to_mongo(self, document):
         if document is None:
             return None
 
-        if isinstance(document, (dict, SON, ObjectId, DBRef)):
+        if isinstance(document, (dict, ObjectId, DBRef)):
             return document
 
         id_field_name = document.__class__._meta["id_field"]
@@ -1539,7 +1539,7 @@ def to_mongo(self, document):
         id_ = id_field.to_mongo(id_)
         collection = document._get_collection_name()
         ref = DBRef(collection, id_)
-        return SON((("_cls", document._class_name), ("_ref", ref)))
+        return {"_cls": document._class_name, "_ref": ref}
 
     def prepare_query_value(self, op, value):
         if value is None:
@@ -2575,7 +2575,7 @@ def build_lazyref(self, value):
                     value.document_type, value.pk, passthrough=self.passthrough
                 )
         elif value is not None:
-            if isinstance(value, (dict, SON)):
+            if isinstance(value, dict):
                 value = LazyReference(
                     _DocumentRegistry.get(value["_cls"]),
                     value["_ref"].id,
@@ -2611,17 +2611,12 @@ def to_mongo(self, document):
             return None
 
         if isinstance(document, LazyReference):
-            return SON(
-                (
-                    ("_cls", document.document_type._class_name),
-                    (
-                        "_ref",
-                        DBRef(
-                            document.document_type._get_collection_name(), document.pk
-                        ),
-                    ),
-                )
-            )
+            return {
+                "_cls": document.document_type._class_name,
+                "_ref": DBRef(
+                    document.document_type._get_collection_name(), document.pk
+                ),
+            }
         else:
             return super().to_mongo(document)
 

mongoengine/queryset/base.py

@@ -6,7 +6,7 @@
 
 import pymongo
 import pymongo.errors
-from bson import SON, json_util
+from bson import json_util
 from bson.code import Code
 from pymongo.collection import ReturnDocument
 from pymongo.common import validate_read_preference
@@ -246,7 +246,7 @@ def search_text(self, text, language=None, text_score=True):
         if queryset._search_text:
             raise OperationError("It is not possible to use search_text two times.")
 
-        query_kwargs = SON({"$search": text})
+        query_kwargs = {"$search": text}
         if language:
             query_kwargs["$language"] = language
 
@@ -1509,7 +1509,7 @@ def map_reduce(
                     if value:
                         ordered_output.append((part, value))
 
-                mr_args["out"] = SON(ordered_output)
+                mr_args["out"] = dict(ordered_output)
 
         db = queryset._document._get_db()
         result = db.command(

mongoengine/queryset/transform.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 
 import pymongo
-from bson import SON, ObjectId
+from bson import ObjectId
 from bson.dbref import DBRef
 
 from mongoengine.base import UPDATE_OPERATORS
@@ -201,38 +201,38 @@ def query(_doc_cls=None, **kwargs):
         else:
             if isinstance(mongo_query[key], dict) and isinstance(value, dict):
                 mongo_query[key].update(value)
-                # $max/minDistance needs to come last - convert to SON
+                # $max/minDistance needs to come last - rebuild in order
                 value_dict = mongo_query[key]
                 if ("$maxDistance" in value_dict or "$minDistance" in value_dict) and (
                     "$near" in value_dict or "$nearSphere" in value_dict
                 ):
-                    value_son = SON()
+                    ordered_value = {}
                     for k, v in value_dict.items():
                         if k == "$maxDistance" or k == "$minDistance":
                             continue
-                        value_son[k] = v
+                        ordered_value[k] = v
                     # Required for MongoDB >= 2.6, may fail when combining
                     # PyMongo 3+ and MongoDB < 2.6
                     near_embedded = False
                     for near_op in ("$near", "$nearSphere"):
                         if isinstance(value_dict.get(near_op), dict):
-                            value_son[near_op] = SON(value_son[near_op])
+                            ordered_value[near_op] = dict(ordered_value[near_op])
                             if "$maxDistance" in value_dict:
-                                value_son[near_op]["$maxDistance"] = value_dict[
+                                ordered_value[near_op]["$maxDistance"] = value_dict[
                                     "$maxDistance"
                                 ]
                             if "$minDistance" in value_dict:
-                                value_son[near_op]["$minDistance"] = value_dict[
+                                ordered_value[near_op]["$minDistance"] = value_dict[
                                     "$minDistance"
                                 ]
                             near_embedded = True
 
                     if not near_embedded:
                         if "$maxDistance" in value_dict:
-                            value_son["$maxDistance"] = value_dict["$maxDistance"]
+                            ordered_value["$maxDistance"] = value_dict["$maxDistance"]
                         if "$minDistance" in value_dict:
-                            value_son["$minDistance"] = value_dict["$minDistance"]
-                    mongo_query[key] = value_son
+                            ordered_value["$minDistance"] = value_dict["$minDistance"]
+                    mongo_query[key] = ordered_value
             else:
                 # Store for manually merging later
                 merge_query[key].append(value)

tests/document/test_delta.py

@@ -1,7 +1,5 @@
 import unittest
 
-from bson import SON
-
 from mongoengine import *
 from mongoengine.pymongo_support import list_collection_names
 from tests.utils import MongoDBTestCase, get_as_pymongo
@@ -663,14 +661,14 @@ class Person(DynamicDocument):
 
         p = Person(name="James", age=34)
         assert p._delta() == (
-            SON([("_cls", "Person"), ("name", "James"), ("age", 34)]),
+            {"_cls": "Person", "name": "James", "age": 34},
             {},
         )
 
         p.doc = 123
         del p.doc
         assert p._delta() == (
-            SON([("_cls", "Person"), ("name", "James"), ("age", 34)]),
+            {"_cls": "Person", "name": "James", "age": 34},
             {},
         )