diff --git a/mock_tests/test_batch.py b/mock_tests/test_batch.py index ad7357f04..6ab13cd2b 100644 --- a/mock_tests/test_batch.py +++ b/mock_tests/test_batch.py @@ -1,10 +1,18 @@ +import asyncio +import threading +from queue import Queue from typing import AsyncGenerator, Generator, List import grpc import pytest import pytest_asyncio import weaviate +from weaviate.collections.batch.async_ import _BatchBaseAsync +from weaviate.collections.batch.base import _BatchStreamRequest +from weaviate.collections.batch.grpc_batch import _BatchGRPC +from weaviate.collections.batch.sync import _BatchBaseSync from weaviate.proto.v1 import batch_pb2, weaviate_pb2_grpc +from weaviate.util import _ServerVersion from .conftest import MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC, mock_class, HTTPServer HOW_MANY = 1000 @@ -160,3 +168,120 @@ def test_ssb_stream_reports_has_errors( batch.add_object({"name": f"Object {i}"}) assert len(failed_object_stream.batch.failed_objects) == 2 assert failed_object_stream.batch.results.objs.has_errors + + +def _make_sync_batch() -> _BatchBaseSync: + # build a _BatchBaseSync without a connection, setting only the fields + # __send and __put_back read, so the stale-sender race can be driven + # deterministically without a server + batch = object.__new__(_BatchBaseSync) + batch._BatchBaseSync__batch_grpc = _BatchGRPC(_ServerVersion.from_string("1.30.0"), None, None) # type: ignore[attr-defined] + batch._BatchBaseSync__is_gcp_on_wcd = False # type: ignore[attr-defined] + batch._BatchBaseSync__bg_exception = None # type: ignore[attr-defined] + batch._BatchBaseSync__reqs = Queue(maxsize=1) # type: ignore[attr-defined] + batch._BatchBaseSync__is_hungup = threading.Event() # type: ignore[attr-defined] + batch._BatchBaseSync__is_shutting_down = threading.Event() # type: ignore[attr-defined] + batch._BatchBaseSync__is_oom = threading.Event() # type: ignore[attr-defined] + batch._BatchBaseSync__shutdown_loop = threading.Event() # type: ignore[attr-defined] + batch._BatchBaseSync__acks_lock = threading.Lock() # type: ignore[attr-defined] + batch._BatchBaseSync__inflight_objs = set() # type: ignore[attr-defined] + batch._BatchBaseSync__inflight_refs = set() # type: ignore[attr-defined] + batch._BatchBaseSync__stream_generation = 0 # type: ignore[attr-defined] + batch._BatchBaseSync__generation_lock = threading.Lock() # type: ignore[attr-defined] + return batch + + +def _stream_request(uuid: str) -> _BatchStreamRequest: + return _BatchStreamRequest(proto=batch_pb2.BatchStreamRequest(), uuids={uuid}, beacons=set()) + + +def test_sync_stale_sender_returns_request_after_fast_reconnect() -> None: + # regression test for #2261: after a hang-up with a sub-second reconnect, + # the old stream's sender must not yield the next request into the dead + # call (gRPC drops it and its uuids wedge __inflight_objs, hanging + # add_object); it must hand the request back to the live stream's sender + batch = _make_sync_batch() + batch._BatchBaseSync__stream_generation = 1 # type: ignore[attr-defined] + stale = batch._BatchBaseSync__send(1) # type: ignore[attr-defined] + assert next(stale).HasField("start") + + # fast reconnect: __recv starts a new stream (generation 2) and clears the + # hung-up flag before the old sender wakes from its queue wait + batch._BatchBaseSync__stream_generation = 2 # type: ignore[attr-defined] + req = _stream_request("uuid-1") + batch._BatchBaseSync__reqs.put(req) # type: ignore[attr-defined] + + with pytest.raises(StopIteration): + next(stale) + # the request is back on the queue and nothing was marked inflight + assert batch._BatchBaseSync__reqs.get_nowait() is req # type: ignore[attr-defined] + assert batch._BatchBaseSync__inflight_objs == set() # type: ignore[attr-defined] + + # the live stream's sender delivers the same request exactly once + batch._BatchBaseSync__reqs.put(req) # type: ignore[attr-defined] + live = batch._BatchBaseSync__send(2) # type: ignore[attr-defined] + assert next(live).HasField("start") + assert next(live) is req.proto + assert batch._BatchBaseSync__inflight_objs == {"uuid-1"} # type: ignore[attr-defined] + + +def test_sync_stale_sender_returns_stop_sentinel_after_fast_reconnect() -> None: + # the None sentinel must also be handed back: if the stale sender consumed + # it, the live stream would never receive its stop message (#2261) + batch = _make_sync_batch() + batch._BatchBaseSync__stream_generation = 1 # type: ignore[attr-defined] + stale = batch._BatchBaseSync__send(1) # type: ignore[attr-defined] + assert next(stale).HasField("start") + + batch._BatchBaseSync__stream_generation = 2 # type: ignore[attr-defined] + batch._BatchBaseSync__reqs.put(None) # type: ignore[attr-defined] + + with pytest.raises(StopIteration): + next(stale) + assert batch._BatchBaseSync__reqs.get_nowait() is None # type: ignore[attr-defined] + + batch._BatchBaseSync__reqs.put(None) # type: ignore[attr-defined] + live = batch._BatchBaseSync__send(2) # type: ignore[attr-defined] + assert next(live).HasField("start") + assert next(live).HasField("stop") + + +def _make_async_batch() -> _BatchBaseAsync: + # async twin of _make_sync_batch, for _BatchBaseAsync.__send/__put_back + batch = object.__new__(_BatchBaseAsync) + batch._BatchBaseAsync__batch_grpc = _BatchGRPC(_ServerVersion.from_string("1.30.0"), None, None) # type: ignore[attr-defined] + batch._BatchBaseAsync__is_gcp_on_wcd = False # type: ignore[attr-defined] + batch._BatchBaseAsync__bg_exception = None # type: ignore[attr-defined] + batch._BatchBaseAsync__reqs = asyncio.Queue(maxsize=1) # type: ignore[attr-defined] + batch._BatchBaseAsync__is_hungup = asyncio.Event() # type: ignore[attr-defined] + batch._BatchBaseAsync__is_shutting_down = asyncio.Event() # type: ignore[attr-defined] + batch._BatchBaseAsync__is_oom = asyncio.Event() # type: ignore[attr-defined] + batch._BatchBaseAsync__shutdown_loop = asyncio.Event() # type: ignore[attr-defined] + batch._BatchBaseAsync__inflight_objs = set() # type: ignore[attr-defined] + batch._BatchBaseAsync__inflight_refs = set() # type: ignore[attr-defined] + batch._BatchBaseAsync__stream_generation = 0 # type: ignore[attr-defined] + return batch + + +@pytest.mark.asyncio +async def test_async_stale_sender_returns_request_after_fast_reconnect() -> None: + # regression test for #2261, async variant of the sync test above + batch = _make_async_batch() + batch._BatchBaseAsync__stream_generation = 1 # type: ignore[attr-defined] + stale = batch._BatchBaseAsync__send(1) # type: ignore[attr-defined] + assert (await anext(stale)).HasField("start") + + batch._BatchBaseAsync__stream_generation = 2 # type: ignore[attr-defined] + req = _stream_request("uuid-1") + batch._BatchBaseAsync__reqs.put_nowait(req) # type: ignore[attr-defined] + + with pytest.raises(StopAsyncIteration): + await anext(stale) + assert batch._BatchBaseAsync__reqs.get_nowait() is req # type: ignore[attr-defined] + assert batch._BatchBaseAsync__inflight_objs == set() # type: ignore[attr-defined] + + batch._BatchBaseAsync__reqs.put_nowait(req) # type: ignore[attr-defined] + live = batch._BatchBaseAsync__send(2) # type: ignore[attr-defined] + assert (await anext(live)).HasField("start") + assert (await anext(live)) is req.proto + assert batch._BatchBaseAsync__inflight_objs == {"uuid-1"} # type: ignore[attr-defined] diff --git a/weaviate/collections/batch/async_.py b/weaviate/collections/batch/async_.py index 07083fe9b..1cc678aa1 100644 --- a/weaviate/collections/batch/async_.py +++ b/weaviate/collections/batch/async_.py @@ -120,6 +120,11 @@ def __init__( # thereby using too much buffer in case of server-side shutdown self.__reqs: asyncio.Queue[Optional[_BatchStreamRequest]] = asyncio.Queue(maxsize=1) + # bumped by __recv every time a new stream is started; a __send generator + # whose generation is no longer current belongs to a dead stream and must + # not consume requests from the queue (#2261) + self.__stream_generation = 0 + self.__bg_exception: Optional[Exception] = None self.__bg_tasks: Optional[_BgTasks] = None @@ -328,7 +333,20 @@ def request_maker(): if len(request.data.objects.values) > 0 or len(request.data.references.values) > 0: yield _BatchStreamRequest(request, uuids, beacons) - async def __send(self) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]: + async def __put_back(self, req: Optional[_BatchStreamRequest]) -> None: + # return a request taken by a stale sender to the queue so the live + # stream's sender can deliver it; the queue has maxsize=1 and this sender + # just freed its slot, but the loop may have refilled it in the meantime, + # so retry until there is room or the batch is being torn down anyway + while self.__bg_exception is None and not self.__shutdown_loop.is_set(): + if not self.__reqs.full(): + # no await between the check and the put, so no other task can + # take the freed slot in between + self.__reqs.put_nowait(req) + return + await asyncio.sleep(0.01) + + async def __send(self, generation: int) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]: yield batch_pb2.BatchStreamRequest( start=batch_pb2.BatchStreamRequest.Start( consistency_level=self.__batch_grpc._consistency_level, @@ -347,6 +365,17 @@ async def __send(self) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]: return try: req = await asyncio.wait_for(self.__reqs.get(), timeout=1) + if generation != self.__stream_generation or self.__is_hungup.is_set(): + # this sender belongs to a stream that hung up and may already + # have been replaced by a fast reconnect, which clears the + # hung-up flag before this sender wakes: do not yield the + # request into the dead call, where gRPC drops it and its + # uuids stay in __inflight_objs forever (#2261) + logger.info( + "Stale batch sender returning its request to the queue for the live stream" + ) + await self.__put_back(req) + return if req is None: logger.info( "Batching finished, stopping and closing the client-side of the stream" @@ -371,12 +400,14 @@ async def __send(self) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]: logger.info("Batch send thread exiting due to exception...") async def __recv(self) -> None: + self.__stream_generation += 1 + generation = self.__stream_generation self.__is_renewing_stream.clear() self.__is_shutting_down.clear() self.__is_hungup.clear() async for message in self.__batch_grpc.astream( connection=self.__connection, - requests=self.__send(), + requests=self.__send(generation), ): if message.HasField("started"): logger.info("Batch stream started successfully") @@ -407,6 +438,8 @@ async def __recv(self) -> None: try: async with self.__objs_cache_lock: cached = self.__objs_cache.pop(error.uuid) + async with self.__uuid_lookup_lock: + self.__uuid_lookup.discard(error.uuid) except KeyError: continue err = ErrorObject( diff --git a/weaviate/collections/batch/sync.py b/weaviate/collections/batch/sync.py index 6cf8c1edc..0567fc8e9 100644 --- a/weaviate/collections/batch/sync.py +++ b/weaviate/collections/batch/sync.py @@ -110,6 +110,12 @@ def __init__( # thereby using too much buffer in case of server-side shutdown self.__reqs: Queue[Optional[_BatchStreamRequest]] = Queue(maxsize=1) + # bumped by __recv every time a new stream is started; a __send generator + # whose generation is no longer current belongs to a dead stream and must + # not consume requests from the queue (#2261) + self.__stream_generation = 0 + self.__generation_lock = threading.Lock() + @property def number_errors(self) -> int: """Return the number of errors in the batch.""" @@ -271,9 +277,23 @@ def request_maker(): if len(request.data.objects.values) > 0 or len(request.data.references.values) > 0: yield _BatchStreamRequest(request, uuids, beacons) - def __send( - self, - ) -> Generator[batch_pb2.BatchStreamRequest, None, None]: + def __current_generation(self) -> int: + with self.__generation_lock: + return self.__stream_generation + + def __put_back(self, req: Optional[_BatchStreamRequest]) -> None: + # return a request taken by a stale sender to the queue so the live + # stream's sender can deliver it; the queue has maxsize=1 and this sender + # just freed its slot, but __loop may have refilled it in the meantime, + # so retry until there is room or the batch is being torn down anyway + while self.__bg_exception is None and not self.__shutdown_loop.is_set(): + try: + self.__reqs.put(req, timeout=0.1) + return + except Full: + continue + + def __send(self, generation: int) -> Generator[batch_pb2.BatchStreamRequest, None, None]: yield batch_pb2.BatchStreamRequest( start=batch_pb2.BatchStreamRequest.Start( consistency_level=self.__batch_grpc._consistency_level, @@ -292,6 +312,17 @@ def __send( return try: req = self.__reqs.get(timeout=1) + if generation != self.__current_generation() or self.__is_hungup.is_set(): + # this sender belongs to a stream that hung up and may already + # have been replaced by a fast reconnect, which clears the + # hung-up flag before this sender wakes: do not yield the + # request into the dead call, where gRPC drops it and its + # uuids stay in __inflight_objs forever (#2261) + logger.info( + "Stale batch sender returning its request to the queue for the live stream" + ) + self.__put_back(req) + return if req is None: logger.info( "Batching finished, stopping and closing the client-side of the stream" @@ -319,12 +350,15 @@ def __send( logger.info("Batch send thread exiting due to exception...") def __recv(self) -> None: + with self.__generation_lock: + self.__stream_generation += 1 + generation = self.__stream_generation self.__is_renewing_stream.clear() self.__is_shutting_down.clear() self.__is_hungup.clear() for message in self.__batch_grpc.stream( connection=self.__connection, - requests=self.__send(), + requests=self.__send(generation), ): if message.HasField("started"): logger.info("Batch stream started successfully") @@ -356,6 +390,8 @@ def __recv(self) -> None: try: with self.__objs_cache_lock: cached = self.__objs_cache.pop(error.uuid) + with self.__uuid_lookup_lock: + self.__uuid_lookup.discard(error.uuid) except KeyError: continue err = ErrorObject(