From 8da8cdc97544ef0041928badb070f1d2952e403b Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:46:25 -0500 Subject: [PATCH 1/8] fix(python): Fix InMemoryCollection upsert with key storage_name alias --- python/semantic_kernel/connectors/in_memory.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/semantic_kernel/connectors/in_memory.py b/python/semantic_kernel/connectors/in_memory.py index f2e1dc30fbe1..ee0876583abe 100644 --- a/python/semantic_kernel/connectors/in_memory.py +++ b/python/semantic_kernel/connectors/in_memory.py @@ -629,12 +629,18 @@ async def _inner_get( return [self.inner_storage[key] for key in keys if key in self.inner_storage] @override - async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[TKey]: + async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[TKey]: updated_keys = [] + key_storage_name = self._key_field_storage_name + key_name = self._key_field_name for record in records: record = AttributeDict(record) - self.inner_storage[record[self._key_field_name]] = record - updated_keys.append(record[self._key_field_name]) + # Serialized records index the key by its storage name (e.g. dict and + # dataclass records), while some record types (e.g. Pydantic models) + # serialize with the Python field name, so fall back to that. + key = record[key_storage_name] if key_storage_name in record else record[key_name] + self.inner_storage[key] = record + updated_keys.append(key) return updated_keys def _deserialize_store_models_to_dicts(self, records: Sequence[Any], **kwargs: Any) -> Sequence[dict[str, Any]]: From 00c355fae4a773ca6c0cefe7ecffaf29c2aae4ee Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:53:48 -0500 Subject: [PATCH 2/8] test(python): add regression tests for upsert with key storage_name alias --- .../unit/connectors/memory/test_in_memory.py | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/python/tests/unit/connectors/memory/test_in_memory.py b/python/tests/unit/connectors/memory/test_in_memory.py index 7955b7696632..b3239d1df60e 100644 --- a/python/tests/unit/connectors/memory/test_in_memory.py +++ b/python/tests/unit/connectors/memory/test_in_memory.py @@ -1,12 +1,14 @@ # Copyright (c) Microsoft. All rights reserved. import ast +from dataclasses import dataclass from pytest import fixture, mark, raises +from pydantic import BaseModel from semantic_kernel.connectors.in_memory import InMemoryCollection, InMemoryStore from semantic_kernel.data._shared import default_dynamic_filter_function -from semantic_kernel.data.vector import DistanceFunction +from semantic_kernel.data.vector import DistanceFunction, VectorStoreCollectionDefinition, VectorStoreField from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreOperationException @@ -292,3 +294,50 @@ async def test_large_sequence_repeat_filter_is_blocked(collection): with raises(VectorStoreOperationException, match="Sequence repetition in filter expressions exceeds the maximum"): collection._get_filtered_records(type("opt", (), {"filter": "lambda x: [0] * 2000000000"})()) + + +@fixture +def storage_name_definition(): + return VectorStoreCollectionDefinition( + fields=[ + VectorStoreField("key", name="id", type="str", storage_name="stored_id"), + VectorStoreField("vector", name="vector", type="float", dimensions=2), + ] + ) + + +async def test_upsert_with_key_storage_name_dict(storage_name_definition): + collection = InMemoryCollection(collection_name="test", record_type=dict, definition=storage_name_definition) + key = await collection.upsert({"id": "first", "vector": [1.0, 0.0]}) + assert key == "first" + result = await collection.get("first") + assert result["id"] == "first" + await collection.delete("first") + assert collection.inner_storage == {} + + +async def test_upsert_with_key_storage_name_dataclass(storage_name_definition): + @dataclass + class DataRecord: + id: str + vector: list[float] + + collection = InMemoryCollection(collection_name="test", record_type=DataRecord, definition=storage_name_definition) + key = await collection.upsert(DataRecord(id="first", vector=[1.0, 0.0])) + assert key == "first" + result = await collection.get("first", include_vectors=True) + assert result.id == "first" + + +async def test_upsert_with_key_storage_name_pydantic(storage_name_definition): + class PydanticRecord(BaseModel): + id: str + vector: list[float] + + collection = InMemoryCollection( + collection_name="test", record_type=PydanticRecord, definition=storage_name_definition + ) + key = await collection.upsert(PydanticRecord(id="first", vector=[1.0, 0.0])) + assert key == "first" + result = await collection.get("first", include_vectors=True) + assert result.id == "first" From df22de73c988f2b039f5fae848075f9e025ab1f9 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:20:16 -0500 Subject: [PATCH 3/8] fix(python): correct indentation of _inner_upsert (review feedback) --- python/semantic_kernel/connectors/in_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/semantic_kernel/connectors/in_memory.py b/python/semantic_kernel/connectors/in_memory.py index ee0876583abe..7f6a6daba04b 100644 --- a/python/semantic_kernel/connectors/in_memory.py +++ b/python/semantic_kernel/connectors/in_memory.py @@ -629,7 +629,7 @@ async def _inner_get( return [self.inner_storage[key] for key in keys if key in self.inner_storage] @override - async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[TKey]: + async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[TKey]: updated_keys = [] key_storage_name = self._key_field_storage_name key_name = self._key_field_name From 937b5d5768c60e0a7fa510a7fa05bf8ece391435 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:28:45 -0600 Subject: [PATCH 4/8] Address review: pick upsert key by serializer contract, fix overlapping-name collision --- .../semantic_kernel/connectors/in_memory.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/python/semantic_kernel/connectors/in_memory.py b/python/semantic_kernel/connectors/in_memory.py index 7f6a6daba04b..9e24b1c5533f 100644 --- a/python/semantic_kernel/connectors/in_memory.py +++ b/python/semantic_kernel/connectors/in_memory.py @@ -6,7 +6,7 @@ from typing import Any, ClassVar, Final, Generic, TypeVar, cast from numpy import dot -from pydantic import Field +from pydantic import BaseModel, Field from scipy.spatial.distance import cityblock, cosine, euclidean, hamming, sqeuclidean from typing_extensions import override @@ -633,18 +633,50 @@ async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence updated_keys = [] key_storage_name = self._key_field_storage_name key_name = self._key_field_name + # Which name indexes the key in a serialized record depends on the + # serializer's contract (see VectorStoreRecordHandler._serialize_data_model_to_dict): + # Pydantic models serialize with model_dump(), which keys fields by + # their Python name, while dict and dataclass records are keyed by + # storage name. Preferring one name over the other is wrong when a + # data field's name or storage name overlaps the key's other name, + # so choose based on the record type and only fall back to the + # other name when the expected one is absent. + is_pydantic = isinstance(self.record_type, type) and issubclass(self.record_type, BaseModel) + expected_key_name = key_name if is_pydantic else key_storage_name + fallback_key_name = key_storage_name if is_pydantic else key_name for record in records: record = AttributeDict(record) - # Serialized records index the key by its storage name (e.g. dict and - # dataclass records), while some record types (e.g. Pydantic models) - # serialize with the Python field name, so fall back to that. - key = record[key_storage_name] if key_storage_name in record else record[key_name] + key = record[expected_key_name] if expected_key_name in record else record[fallback_key_name] self.inner_storage[key] = record updated_keys.append(key) return updated_keys def _deserialize_store_models_to_dicts(self, records: Sequence[Any], **kwargs: Any) -> Sequence[dict[str, Any]]: - return records + if not (isinstance(self.record_type, type) and issubclass(self.record_type, BaseModel)): + return records + # Pydantic records are stored under their Python field names (they are + # serialized with model_dump()), but the shared deserializer for + # Pydantic models expects store dicts keyed by storage name and maps + # them back to field names. Rename name -> storage name in one pass + # so a data field whose name overlaps the key's storage name (or vice + # versa) keeps its own value instead of overwriting the key's. + converted_records: list[dict[str, Any]] = [] + for record in records: + renames = { + field.storage_name: record[field.name] + for field in self.definition.fields + if field.storage_name and field.storage_name != field.name and field.name in record + } + if not renames: + converted_records.append(record) + continue + # Drop every renamed field's original name, then apply all + # renames at once so overlapping names cannot overwrite each other. + renamed_names = {field.name for field in self.definition.fields if field.storage_name in renames} + converted = {key: value for key, value in record.items() if key not in renamed_names} + converted.update(renames) + converted_records.append(converted) + return converted_records def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **kwargs: Any) -> Sequence[Any]: return records From 17a93909cc9051405ad098a2166272b3978c1b99 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:29:33 -0600 Subject: [PATCH 5/8] Add overlapping-name regression test and apply Ruff import ordering --- .../unit/connectors/memory/test_in_memory.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/python/tests/unit/connectors/memory/test_in_memory.py b/python/tests/unit/connectors/memory/test_in_memory.py index b3239d1df60e..1cb6e7de0db9 100644 --- a/python/tests/unit/connectors/memory/test_in_memory.py +++ b/python/tests/unit/connectors/memory/test_in_memory.py @@ -3,8 +3,8 @@ import ast from dataclasses import dataclass -from pytest import fixture, mark, raises from pydantic import BaseModel +from pytest import fixture, mark, raises from semantic_kernel.connectors.in_memory import InMemoryCollection, InMemoryStore from semantic_kernel.data._shared import default_dynamic_filter_function @@ -341,3 +341,28 @@ class PydanticRecord(BaseModel): assert key == "first" result = await collection.get("first", include_vectors=True) assert result.id == "first" + + +async def test_upsert_pydantic_key_storage_name_overlapping_data_field(): + # The key field's storage name ("stored_id") is also the name of a data + # field. Pydantic records serialize with logical names, so the key must + # come from "id", not from the data field named "stored_id". + definition = VectorStoreCollectionDefinition( + fields=[ + VectorStoreField("key", name="id", type="str", storage_name="stored_id"), + VectorStoreField("data", name="stored_id", type="str", storage_name="payload"), + VectorStoreField("vector", name="vector", type="float", dimensions=2), + ] + ) + + class OverlappingRecord(BaseModel): + id: str + stored_id: str + vector: list[float] + + collection = InMemoryCollection(collection_name="test", record_type=OverlappingRecord, definition=definition) + key = await collection.upsert(OverlappingRecord(id="actual-key", stored_id="payload-value", vector=[1.0, 0.0])) + assert key == "actual-key" + result = await collection.get("actual-key", include_vectors=True) + assert result.id == "actual-key" + assert result.stored_id == "payload-value" From 35e92048e804613545860350121fb59c1ddf3a8c Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:37:41 -0600 Subject: [PATCH 6/8] Merge upstream main: keep read-side deserialize fix for overlapping names --- .../semantic_kernel/connectors/in_memory.py | 59 +++++++++++++------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/python/semantic_kernel/connectors/in_memory.py b/python/semantic_kernel/connectors/in_memory.py index 9e24b1c5533f..2b899d82c84c 100644 --- a/python/semantic_kernel/connectors/in_memory.py +++ b/python/semantic_kernel/connectors/in_memory.py @@ -6,7 +6,7 @@ from typing import Any, ClassVar, Final, Generic, TypeVar, cast from numpy import dot -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from scipy.spatial.distance import cityblock, cosine, euclidean, hamming, sqeuclidean from typing_extensions import override @@ -27,7 +27,7 @@ ) from semantic_kernel.exceptions import VectorSearchExecutionException, VectorStoreModelValidationError from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreModelException, VectorStoreOperationException -from semantic_kernel.kernel_types import OneOrMany +from semantic_kernel.kernel_types import OneOrList, OneOrMany from semantic_kernel.utils.feature_stage_decorator import release_candidate from semantic_kernel.utils.list_handler import empty_generator @@ -81,6 +81,15 @@ def __delattr__(self, name) -> None: raise AttributeError(name) +class _InMemoryRecord(dict[str, Any]): + """A serialized record with the name of its key field.""" + + def __init__(self, record: dict[str, Any], key_field_name: str): + """Keep the dumped fields and the name of their key field.""" + super().__init__(record) + self.key_field_name = key_field_name + + class ReadOnlyAttributeDict(Mapping[TAKey, TAValue], Generic[TAKey, TAValue]): """A read-only mapping that allows attribute access to keys.""" @@ -631,22 +640,16 @@ async def _inner_get( @override async def _inner_upsert(self, records: Sequence[Any], **kwargs: Any) -> Sequence[TKey]: updated_keys = [] - key_storage_name = self._key_field_storage_name - key_name = self._key_field_name - # Which name indexes the key in a serialized record depends on the - # serializer's contract (see VectorStoreRecordHandler._serialize_data_model_to_dict): - # Pydantic models serialize with model_dump(), which keys fields by - # their Python name, while dict and dataclass records are keyed by - # storage name. Preferring one name over the other is wrong when a - # data field's name or storage name overlaps the key's other name, - # so choose based on the record type and only fall back to the - # other name when the expected one is absent. - is_pydantic = isinstance(self.record_type, type) and issubclass(self.record_type, BaseModel) - expected_key_name = key_name if is_pydantic else key_storage_name - fallback_key_name = key_storage_name if is_pydantic else key_name for record in records: + key_field_name = ( + record.key_field_name if isinstance(record, _InMemoryRecord) else self._key_field_storage_name + ) record = AttributeDict(record) - key = record[expected_key_name] if expected_key_name in record else record[fallback_key_name] + if key_field_name not in record: + key_field_name = ( + self._key_field_storage_name if self._key_field_storage_name in record else self._key_field_name + ) + key = record[key_field_name] self.inner_storage[key] = record updated_keys.append(key) return updated_keys @@ -678,6 +681,25 @@ def _deserialize_store_models_to_dicts(self, records: Sequence[Any], **kwargs: A converted_records.append(converted) return converted_records + def _serialize_data_model_to_dict(self, record: TModel, **kwargs: Any) -> OneOrList[dict[str, Any]]: + """Keep track of key names before records lose their model type.""" + serialized = super()._serialize_data_model_to_dict(record, **kwargs) + key_field_name = self._key_field_name + if not self.definition.to_dict: + if not isinstance(record, BaseModel): + key_field_name = self._key_field_storage_name + elif ( + type(record).model_dump is BaseModel.model_dump + and not type(record).__pydantic_decorators__.model_serializers + and "serialize_by_alias" in ConfigDict.__annotations__ + and type(record).model_config.get("serialize_by_alias") + and (key_field := type(record).model_fields.get(key_field_name)) + ): + key_field_name = key_field.serialization_alias or key_field_name + if isinstance(serialized, list): + return [_InMemoryRecord(item, key_field_name) for item in serialized] + return _InMemoryRecord(serialized, key_field_name) + def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **kwargs: Any) -> Sequence[Any]: return records @@ -773,9 +795,8 @@ def _get_filtered_records(self, options: VectorSearchOptions) -> dict[TKey, Attr raise VectorStoreOperationException(f"Error evaluating filter: {e}") from e filtered_records: dict[TKey, AttributeDict] = {} for key, record in self.inner_storage.items(): - for filter in callable_filters: - if self._run_filter(filter, record): - filtered_records[key] = record + if all(self._run_filter(filter, record) for filter in callable_filters): + filtered_records[key] = record return filtered_records def _parse_and_validate_filter(self, filter_str: str) -> Callable: From 7b5d523d61b2adbf544695eb288e5db337de4a38 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:38:28 -0600 Subject: [PATCH 7/8] Rebase tests onto upstream storage-name suite, keep get round-trip regression --- .../unit/connectors/memory/test_in_memory.py | 275 ++++++++++++++---- 1 file changed, 226 insertions(+), 49 deletions(-) diff --git a/python/tests/unit/connectors/memory/test_in_memory.py b/python/tests/unit/connectors/memory/test_in_memory.py index 1cb6e7de0db9..779f62a6b5a6 100644 --- a/python/tests/unit/connectors/memory/test_in_memory.py +++ b/python/tests/unit/connectors/memory/test_in_memory.py @@ -1,9 +1,8 @@ # Copyright (c) Microsoft. All rights reserved. import ast -from dataclasses import dataclass -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field, model_serializer from pytest import fixture, mark, raises from semantic_kernel.connectors.in_memory import InMemoryCollection, InMemoryStore @@ -37,6 +36,209 @@ async def test_upsert(collection): assert collection.inner_storage == {"testid": record} +@mark.parametrize("batch", [False, True]) +@mark.parametrize("model_kind", ["dict", "dataclass", "pydantic"]) +async def test_upsert_with_key_storage_name(definition, dataclass_vector_data_model, record_type, batch, model_kind): + definition.key_field.storage_name = "stored_id" + model_type = {"dict": dict, "dataclass": dataclass_vector_data_model, "pydantic": record_type}[model_kind] + collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition) + records = [ + model_type(id="first", content="first content", vector=[1.0, 0.0, 0.0, 0.0, 0.0]), + model_type(id="second", content="second content", vector=[0.0, 1.0, 0.0, 0.0, 0.0]), + ] + + keys = await collection.upsert(records if batch else records[0]) + assert keys == (["first", "second"] if batch else "first") + assert await collection.get("first", include_vectors=True) == records[0] + if batch: + assert await collection.get(["first", "second"], include_vectors=True) == records + + results = await collection.search(vector=[1.0, 0.0, 0.0, 0.0, 0.0], include_vectors=True) + found = [result.record async for result in results.results] + assert found == (records if batch else records[:1]) + + await collection.delete("first") + assert await collection.get("first") is None + if batch: + assert await collection.get("second", include_vectors=True) == records[1] + + +@mark.parametrize("batch", [False, True]) +async def test_upsert_when_storage_name_matches_logical_field(definition, record_type, batch): + definition.key_field.storage_name = "content" + collection = InMemoryCollection(collection_name="test", record_type=record_type, definition=definition) + record = record_type(id="first", content="unrelated", vector=[1.0, 0.0, 0.0, 0.0, 0.0]) + + keys = await collection.upsert([record] if batch else record) + assert keys == (["first"] if batch else "first") + assert "first" in collection.inner_storage + assert "unrelated" not in collection.inner_storage + + await collection.delete("first") + assert collection.inner_storage == {} + + +@mark.parametrize("batch", [False, True]) +@mark.parametrize("serializer", ["default", "serialize_none", "model_serialize_none"]) +async def test_upsert_when_key_storage_name_matches_another_logical_field(definition, record_type, batch, serializer): + definition.key_field.storage_name = "content" + if serializer == "serialize_none": + definition.serialize = lambda record, **kwargs: None + elif serializer == "model_serialize_none": + + class CustomRecord(record_type): + def serialize(self, **kwargs): + return None + + record_type = CustomRecord + collection = InMemoryCollection(collection_name="test", record_type=record_type, definition=definition) + records = [ + record_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]), + record_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]), + ] + + keys = await collection.upsert(records if batch else records[0]) + assert keys == (["first", "second"] if batch else "first") + assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"}) + assert collection.inner_storage["first"] == records[0].model_dump() + + await collection.delete(["first", "second"] if batch else "first") + assert collection.inner_storage == {} + + +@mark.parametrize("batch", [False, True]) +@mark.parametrize("model_kind", ["dict", "dataclass"]) +async def test_upsert_when_another_storage_name_matches_key(definition, dataclass_vector_data_model, batch, model_kind): + definition.key_field.storage_name = "stored_id" + definition.fields[1].storage_name = "id" + model_type = dict if model_kind == "dict" else dataclass_vector_data_model + collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition) + records = [ + model_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]), + model_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]), + ] + + keys = await collection.upsert(records if batch else records[0]) + assert keys == (["first", "second"] if batch else "first") + assert await collection.get("first", include_vectors=True) == records[0] + if batch: + assert await collection.get(["first", "second"], include_vectors=True) == records + + await collection.delete(["first", "second"] if batch else "first") + assert collection.inner_storage == {} + + +@mark.parametrize("batch", [False, True]) +@mark.parametrize("serializer", ["to_dict", "serialize", "model_serialize", "model_dump", "model_serializer"]) +async def test_upsert_with_pydantic_custom_storage_serialization(definition, record_type, batch, serializer): + definition.key_field.storage_name = "stored_id" + content_name = "id" if serializer in {"serialize", "model_serialize"} else "stored_content" + definition.fields[1].storage_name = content_name + if serializer == "model_serialize": + + class CustomRecord(record_type): + def serialize(self, **kwargs): + return {"stored_id": self.id, content_name: self.content, "vector": self.vector} + + model_type = CustomRecord + elif serializer == "model_dump": + + class CustomDumpRecord(record_type): + def model_dump(self, **kwargs): + return {"stored_id": self.id, content_name: self.content, "vector": self.vector} + + model_type = CustomDumpRecord + elif serializer == "model_serializer": + + class CustomSerializedRecord(record_type): + @model_serializer + def serialize_model(self): + return {"stored_id": self.id, content_name: self.content, "vector": self.vector} + + model_type = CustomSerializedRecord + else: + model_type = record_type + setattr( + definition, + serializer, + lambda record, **kwargs: {"stored_id": record.id, content_name: record.content, "vector": record.vector}, + ) + collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition) + records = [ + model_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]), + model_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]), + ] + + keys = await collection.upsert(records if batch else records[0]) + assert keys == (["first", "second"] if batch else "first") + assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"}) + + await collection.delete(["first", "second"] if batch else "first") + assert collection.inner_storage == {} + + +@mark.parametrize("batch", [False, True]) +async def test_upsert_with_pydantic_serialization_aliases(definition, record_type, batch): + definition.key_field.storage_name = "stored_id" + definition.fields[1].storage_name = "id" + + class AliasedRecord(record_type): + model_config = ConfigDict(serialize_by_alias=True) + id: str = Field(serialization_alias="stored_id") + content: str = Field(serialization_alias="id") + + collection = InMemoryCollection(collection_name="test", record_type=AliasedRecord, definition=definition) + records = [ + AliasedRecord(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]), + AliasedRecord(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]), + ] + + keys = await collection.upsert(records if batch else records[0]) + assert keys == (["first", "second"] if batch else "first") + assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"}) + + await collection.delete(["first", "second"] if batch else "first") + assert collection.inner_storage == {} + + +@mark.parametrize("batch", [False, True]) +@mark.parametrize("serializer", ["model_dump", "model_serializer", "to_dict"]) +async def test_upsert_with_custom_logical_serialization(definition, record_type, batch, serializer): + definition.key_field.storage_name = "content" + definition.fields[1].storage_name = "id" + if serializer == "model_dump": + + class CustomRecord(record_type): + def model_dump(self, **kwargs): + return super().model_dump(**kwargs) + + model_type = CustomRecord + elif serializer == "model_serializer": + + class CustomRecord(record_type): + @model_serializer(mode="wrap") + def serialize_model(self, handler): + return handler(self) + + model_type = CustomRecord + else: + model_type = record_type + definition.to_dict = lambda record, **kwargs: record.model_dump() + + collection = InMemoryCollection(collection_name="test", record_type=model_type, definition=definition) + records = [ + model_type(id="first", content="unrelated first", vector=[1.0, 0.0, 0.0, 0.0, 0.0]), + model_type(id="second", content="unrelated second", vector=[0.0, 1.0, 0.0, 0.0, 0.0]), + ] + + keys = await collection.upsert(records if batch else records[0]) + assert keys == (["first", "second"] if batch else "first") + assert set(collection.inner_storage) == ({"first", "second"} if batch else {"first"}) + + await collection.delete(["first", "second"] if batch else "first") + assert collection.inner_storage == {} + + async def test_get(collection): record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]} await collection.upsert(record) @@ -125,6 +327,28 @@ async def test_valid_lambda_filter_attribute_access(collection): assert "2" in results +async def test_multiple_filters_are_combined_with_and(collection): + await collection.upsert([ + {"id": "1", "content": "a", "vector": [1, 2, 3, 4, 5]}, + {"id": "2", "content": "b", "vector": [1, 2, 3, 4, 5]}, + {"id": "3", "content": "a", "vector": [1, 2, 3, 4, 5]}, + ]) + filters = ["lambda x: x.content == 'a'", "lambda x: x.id != '3'"] + results = collection._get_filtered_records(type("opt", (), {"filter": filters})()) + assert list(results) == ["1"] + + +async def test_multiple_callable_filters_are_combined_with_and(collection): + await collection.upsert([ + {"id": "1", "content": "a", "vector": [1, 2, 3, 4, 5]}, + {"id": "2", "content": "b", "vector": [1, 2, 3, 4, 5]}, + {"id": "3", "content": "a", "vector": [1, 2, 3, 4, 5]}, + ]) + filters = [lambda x: x.content == "a", lambda x: x.id != "3"] + results = collection._get_filtered_records(type("opt", (), {"filter": filters})()) + assert list(results) == ["1"] + + async def test_invalid_filter_not_lambda(collection): with raises(VectorStoreOperationException, match="must be a lambda expression"): collection._get_filtered_records(type("opt", (), {"filter": "x.id == '1'"})()) @@ -296,53 +520,6 @@ async def test_large_sequence_repeat_filter_is_blocked(collection): collection._get_filtered_records(type("opt", (), {"filter": "lambda x: [0] * 2000000000"})()) -@fixture -def storage_name_definition(): - return VectorStoreCollectionDefinition( - fields=[ - VectorStoreField("key", name="id", type="str", storage_name="stored_id"), - VectorStoreField("vector", name="vector", type="float", dimensions=2), - ] - ) - - -async def test_upsert_with_key_storage_name_dict(storage_name_definition): - collection = InMemoryCollection(collection_name="test", record_type=dict, definition=storage_name_definition) - key = await collection.upsert({"id": "first", "vector": [1.0, 0.0]}) - assert key == "first" - result = await collection.get("first") - assert result["id"] == "first" - await collection.delete("first") - assert collection.inner_storage == {} - - -async def test_upsert_with_key_storage_name_dataclass(storage_name_definition): - @dataclass - class DataRecord: - id: str - vector: list[float] - - collection = InMemoryCollection(collection_name="test", record_type=DataRecord, definition=storage_name_definition) - key = await collection.upsert(DataRecord(id="first", vector=[1.0, 0.0])) - assert key == "first" - result = await collection.get("first", include_vectors=True) - assert result.id == "first" - - -async def test_upsert_with_key_storage_name_pydantic(storage_name_definition): - class PydanticRecord(BaseModel): - id: str - vector: list[float] - - collection = InMemoryCollection( - collection_name="test", record_type=PydanticRecord, definition=storage_name_definition - ) - key = await collection.upsert(PydanticRecord(id="first", vector=[1.0, 0.0])) - assert key == "first" - result = await collection.get("first", include_vectors=True) - assert result.id == "first" - - async def test_upsert_pydantic_key_storage_name_overlapping_data_field(): # The key field's storage name ("stored_id") is also the name of a data # field. Pydantic records serialize with logical names, so the key must From 72c6c51cd0b0cd3f8436952eeb377343dcd48c68 Mon Sep 17 00:00:00 2001 From: Jeevan Babu Gotru <114808955+jeevan1098@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:47:59 -0600 Subject: [PATCH 8/8] Remove duplicated _serialize_data_model_to_dict left by web merge --- .../semantic_kernel/connectors/in_memory.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/python/semantic_kernel/connectors/in_memory.py b/python/semantic_kernel/connectors/in_memory.py index 5c306f080fab..2b899d82c84c 100644 --- a/python/semantic_kernel/connectors/in_memory.py +++ b/python/semantic_kernel/connectors/in_memory.py @@ -700,25 +700,6 @@ def _serialize_data_model_to_dict(self, record: TModel, **kwargs: Any) -> OneOrL return [_InMemoryRecord(item, key_field_name) for item in serialized] return _InMemoryRecord(serialized, key_field_name) - def _serialize_data_model_to_dict(self, record: TModel, **kwargs: Any) -> OneOrList[dict[str, Any]]: - """Keep track of key names before records lose their model type.""" - serialized = super()._serialize_data_model_to_dict(record, **kwargs) - key_field_name = self._key_field_name - if not self.definition.to_dict: - if not isinstance(record, BaseModel): - key_field_name = self._key_field_storage_name - elif ( - type(record).model_dump is BaseModel.model_dump - and not type(record).__pydantic_decorators__.model_serializers - and "serialize_by_alias" in ConfigDict.__annotations__ - and type(record).model_config.get("serialize_by_alias") - and (key_field := type(record).model_fields.get(key_field_name)) - ): - key_field_name = key_field.serialization_alias or key_field_name - if isinstance(serialized, list): - return [_InMemoryRecord(item, key_field_name) for item in serialized] - return _InMemoryRecord(serialized, key_field_name) - def _serialize_dicts_to_store_models(self, records: Sequence[dict[str, Any]], **kwargs: Any) -> Sequence[Any]: return records