Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 125 additions & 0 deletions mock_tests/test_batch.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
import asyncio
import threading
from queue import Queue
from typing import AsyncGenerator, Generator, List

import grpc
import pytest
import pytest_asyncio
import weaviate
from weaviate.collections.batch.async_ import _BatchBaseAsync
from weaviate.collections.batch.base import _BatchStreamRequest
from weaviate.collections.batch.grpc_batch import _BatchGRPC
from weaviate.collections.batch.sync import _BatchBaseSync
from weaviate.proto.v1 import batch_pb2, weaviate_pb2_grpc
from weaviate.util import _ServerVersion
from .conftest import MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC, mock_class, HTTPServer

HOW_MANY = 1000
Expand Down Expand Up @@ -160,3 +168,120 @@ def test_ssb_stream_reports_has_errors(
batch.add_object({"name": f"Object {i}"})
assert len(failed_object_stream.batch.failed_objects) == 2
assert failed_object_stream.batch.results.objs.has_errors


def _make_sync_batch() -> _BatchBaseSync:
# build a _BatchBaseSync without a connection, setting only the fields
# __send and __put_back read, so the stale-sender race can be driven
# deterministically without a server
batch = object.__new__(_BatchBaseSync)
batch._BatchBaseSync__batch_grpc = _BatchGRPC(_ServerVersion.from_string("1.30.0"), None, None) # type: ignore[attr-defined]
batch._BatchBaseSync__is_gcp_on_wcd = False # type: ignore[attr-defined]
batch._BatchBaseSync__bg_exception = None # type: ignore[attr-defined]
batch._BatchBaseSync__reqs = Queue(maxsize=1) # type: ignore[attr-defined]
batch._BatchBaseSync__is_hungup = threading.Event() # type: ignore[attr-defined]
batch._BatchBaseSync__is_shutting_down = threading.Event() # type: ignore[attr-defined]
batch._BatchBaseSync__is_oom = threading.Event() # type: ignore[attr-defined]
batch._BatchBaseSync__shutdown_loop = threading.Event() # type: ignore[attr-defined]
batch._BatchBaseSync__acks_lock = threading.Lock() # type: ignore[attr-defined]
batch._BatchBaseSync__inflight_objs = set() # type: ignore[attr-defined]
batch._BatchBaseSync__inflight_refs = set() # type: ignore[attr-defined]
batch._BatchBaseSync__stream_generation = 0 # type: ignore[attr-defined]
batch._BatchBaseSync__generation_lock = threading.Lock() # type: ignore[attr-defined]
return batch


def _stream_request(uuid: str) -> _BatchStreamRequest:
return _BatchStreamRequest(proto=batch_pb2.BatchStreamRequest(), uuids={uuid}, beacons=set())


def test_sync_stale_sender_returns_request_after_fast_reconnect() -> None:
# regression test for #2261: after a hang-up with a sub-second reconnect,
# the old stream's sender must not yield the next request into the dead
# call (gRPC drops it and its uuids wedge __inflight_objs, hanging
# add_object); it must hand the request back to the live stream's sender
batch = _make_sync_batch()
batch._BatchBaseSync__stream_generation = 1 # type: ignore[attr-defined]
stale = batch._BatchBaseSync__send(1) # type: ignore[attr-defined]
assert next(stale).HasField("start")

# fast reconnect: __recv starts a new stream (generation 2) and clears the
# hung-up flag before the old sender wakes from its queue wait
batch._BatchBaseSync__stream_generation = 2 # type: ignore[attr-defined]
req = _stream_request("uuid-1")
batch._BatchBaseSync__reqs.put(req) # type: ignore[attr-defined]

with pytest.raises(StopIteration):
next(stale)
# the request is back on the queue and nothing was marked inflight
assert batch._BatchBaseSync__reqs.get_nowait() is req # type: ignore[attr-defined]
assert batch._BatchBaseSync__inflight_objs == set() # type: ignore[attr-defined]

# the live stream's sender delivers the same request exactly once
batch._BatchBaseSync__reqs.put(req) # type: ignore[attr-defined]
live = batch._BatchBaseSync__send(2) # type: ignore[attr-defined]
assert next(live).HasField("start")
assert next(live) is req.proto
assert batch._BatchBaseSync__inflight_objs == {"uuid-1"} # type: ignore[attr-defined]


def test_sync_stale_sender_returns_stop_sentinel_after_fast_reconnect() -> None:
# the None sentinel must also be handed back: if the stale sender consumed
# it, the live stream would never receive its stop message (#2261)
batch = _make_sync_batch()
batch._BatchBaseSync__stream_generation = 1 # type: ignore[attr-defined]
stale = batch._BatchBaseSync__send(1) # type: ignore[attr-defined]
assert next(stale).HasField("start")

batch._BatchBaseSync__stream_generation = 2 # type: ignore[attr-defined]
batch._BatchBaseSync__reqs.put(None) # type: ignore[attr-defined]

with pytest.raises(StopIteration):
next(stale)
assert batch._BatchBaseSync__reqs.get_nowait() is None # type: ignore[attr-defined]

batch._BatchBaseSync__reqs.put(None) # type: ignore[attr-defined]
live = batch._BatchBaseSync__send(2) # type: ignore[attr-defined]
assert next(live).HasField("start")
assert next(live).HasField("stop")


def _make_async_batch() -> _BatchBaseAsync:
# async twin of _make_sync_batch, for _BatchBaseAsync.__send/__put_back
batch = object.__new__(_BatchBaseAsync)
batch._BatchBaseAsync__batch_grpc = _BatchGRPC(_ServerVersion.from_string("1.30.0"), None, None) # type: ignore[attr-defined]
batch._BatchBaseAsync__is_gcp_on_wcd = False # type: ignore[attr-defined]
batch._BatchBaseAsync__bg_exception = None # type: ignore[attr-defined]
batch._BatchBaseAsync__reqs = asyncio.Queue(maxsize=1) # type: ignore[attr-defined]
batch._BatchBaseAsync__is_hungup = asyncio.Event() # type: ignore[attr-defined]
batch._BatchBaseAsync__is_shutting_down = asyncio.Event() # type: ignore[attr-defined]
batch._BatchBaseAsync__is_oom = asyncio.Event() # type: ignore[attr-defined]
batch._BatchBaseAsync__shutdown_loop = asyncio.Event() # type: ignore[attr-defined]
batch._BatchBaseAsync__inflight_objs = set() # type: ignore[attr-defined]
batch._BatchBaseAsync__inflight_refs = set() # type: ignore[attr-defined]
batch._BatchBaseAsync__stream_generation = 0 # type: ignore[attr-defined]
return batch


@pytest.mark.asyncio
async def test_async_stale_sender_returns_request_after_fast_reconnect() -> None:
# regression test for #2261, async variant of the sync test above
batch = _make_async_batch()
batch._BatchBaseAsync__stream_generation = 1 # type: ignore[attr-defined]
stale = batch._BatchBaseAsync__send(1) # type: ignore[attr-defined]
assert (await anext(stale)).HasField("start")

batch._BatchBaseAsync__stream_generation = 2 # type: ignore[attr-defined]
req = _stream_request("uuid-1")
batch._BatchBaseAsync__reqs.put_nowait(req) # type: ignore[attr-defined]

with pytest.raises(StopAsyncIteration):
await anext(stale)
assert batch._BatchBaseAsync__reqs.get_nowait() is req # type: ignore[attr-defined]
assert batch._BatchBaseAsync__inflight_objs == set() # type: ignore[attr-defined]

batch._BatchBaseAsync__reqs.put_nowait(req) # type: ignore[attr-defined]
live = batch._BatchBaseAsync__send(2) # type: ignore[attr-defined]
assert (await anext(live)).HasField("start")
assert (await anext(live)) is req.proto
assert batch._BatchBaseAsync__inflight_objs == {"uuid-1"} # type: ignore[attr-defined]
37 changes: 35 additions & 2 deletions weaviate/collections/batch/async_.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ def __init__(
# thereby using too much buffer in case of server-side shutdown
self.__reqs: asyncio.Queue[Optional[_BatchStreamRequest]] = asyncio.Queue(maxsize=1)

# bumped by __recv every time a new stream is started; a __send generator
# whose generation is no longer current belongs to a dead stream and must
# not consume requests from the queue (#2261)
self.__stream_generation = 0

self.__bg_exception: Optional[Exception] = None
self.__bg_tasks: Optional[_BgTasks] = None

Expand Down Expand Up @@ -328,7 +333,20 @@ def request_maker():
if len(request.data.objects.values) > 0 or len(request.data.references.values) > 0:
yield _BatchStreamRequest(request, uuids, beacons)

async def __send(self) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]:
async def __put_back(self, req: Optional[_BatchStreamRequest]) -> None:
# return a request taken by a stale sender to the queue so the live
# stream's sender can deliver it; the queue has maxsize=1 and this sender
# just freed its slot, but the loop may have refilled it in the meantime,
# so retry until there is room or the batch is being torn down anyway
while self.__bg_exception is None and not self.__shutdown_loop.is_set():
if not self.__reqs.full():
# no await between the check and the put, so no other task can
# take the freed slot in between
self.__reqs.put_nowait(req)
return
await asyncio.sleep(0.01)

async def __send(self, generation: int) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]:
yield batch_pb2.BatchStreamRequest(
start=batch_pb2.BatchStreamRequest.Start(
consistency_level=self.__batch_grpc._consistency_level,
Expand All @@ -347,6 +365,17 @@ async def __send(self) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]:
return
try:
req = await asyncio.wait_for(self.__reqs.get(), timeout=1)
if generation != self.__stream_generation or self.__is_hungup.is_set():
# this sender belongs to a stream that hung up and may already
# have been replaced by a fast reconnect, which clears the
# hung-up flag before this sender wakes: do not yield the
# request into the dead call, where gRPC drops it and its
# uuids stay in __inflight_objs forever (#2261)
logger.info(
"Stale batch sender returning its request to the queue for the live stream"
)
await self.__put_back(req)
return
if req is None:
logger.info(
"Batching finished, stopping and closing the client-side of the stream"
Expand All @@ -371,12 +400,14 @@ async def __send(self) -> AsyncGenerator[batch_pb2.BatchStreamRequest, None]:
logger.info("Batch send thread exiting due to exception...")

async def __recv(self) -> None:
self.__stream_generation += 1
generation = self.__stream_generation
self.__is_renewing_stream.clear()
self.__is_shutting_down.clear()
self.__is_hungup.clear()
async for message in self.__batch_grpc.astream(
connection=self.__connection,
requests=self.__send(),
requests=self.__send(generation),
):
if message.HasField("started"):
logger.info("Batch stream started successfully")
Expand Down Expand Up @@ -407,6 +438,8 @@ async def __recv(self) -> None:
try:
async with self.__objs_cache_lock:
cached = self.__objs_cache.pop(error.uuid)
async with self.__uuid_lookup_lock:
self.__uuid_lookup.discard(error.uuid)
except KeyError:
continue
err = ErrorObject(
Expand Down
44 changes: 40 additions & 4 deletions weaviate/collections/batch/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,12 @@ def __init__(
# thereby using too much buffer in case of server-side shutdown
self.__reqs: Queue[Optional[_BatchStreamRequest]] = Queue(maxsize=1)

# bumped by __recv every time a new stream is started; a __send generator
# whose generation is no longer current belongs to a dead stream and must
# not consume requests from the queue (#2261)
self.__stream_generation = 0
self.__generation_lock = threading.Lock()

@property
def number_errors(self) -> int:
"""Return the number of errors in the batch."""
Expand Down Expand Up @@ -271,9 +277,23 @@ def request_maker():
if len(request.data.objects.values) > 0 or len(request.data.references.values) > 0:
yield _BatchStreamRequest(request, uuids, beacons)

def __send(
self,
) -> Generator[batch_pb2.BatchStreamRequest, None, None]:
def __current_generation(self) -> int:
with self.__generation_lock:
return self.__stream_generation

def __put_back(self, req: Optional[_BatchStreamRequest]) -> None:
# return a request taken by a stale sender to the queue so the live
# stream's sender can deliver it; the queue has maxsize=1 and this sender
# just freed its slot, but __loop may have refilled it in the meantime,
# so retry until there is room or the batch is being torn down anyway
while self.__bg_exception is None and not self.__shutdown_loop.is_set():
try:
self.__reqs.put(req, timeout=0.1)
return
except Full:
continue

def __send(self, generation: int) -> Generator[batch_pb2.BatchStreamRequest, None, None]:
yield batch_pb2.BatchStreamRequest(
start=batch_pb2.BatchStreamRequest.Start(
consistency_level=self.__batch_grpc._consistency_level,
Expand All @@ -292,6 +312,17 @@ def __send(
return
try:
req = self.__reqs.get(timeout=1)
if generation != self.__current_generation() or self.__is_hungup.is_set():
# this sender belongs to a stream that hung up and may already
# have been replaced by a fast reconnect, which clears the
# hung-up flag before this sender wakes: do not yield the
# request into the dead call, where gRPC drops it and its
# uuids stay in __inflight_objs forever (#2261)
logger.info(
"Stale batch sender returning its request to the queue for the live stream"
)
self.__put_back(req)
return
if req is None:
logger.info(
"Batching finished, stopping and closing the client-side of the stream"
Expand Down Expand Up @@ -319,12 +350,15 @@ def __send(
logger.info("Batch send thread exiting due to exception...")

def __recv(self) -> None:
with self.__generation_lock:
self.__stream_generation += 1
generation = self.__stream_generation
self.__is_renewing_stream.clear()
self.__is_shutting_down.clear()
self.__is_hungup.clear()
for message in self.__batch_grpc.stream(
connection=self.__connection,
requests=self.__send(),
requests=self.__send(generation),
):
if message.HasField("started"):
logger.info("Batch stream started successfully")
Expand Down Expand Up @@ -356,6 +390,8 @@ def __recv(self) -> None:
try:
with self.__objs_cache_lock:
cached = self.__objs_cache.pop(error.uuid)
with self.__uuid_lookup_lock:
self.__uuid_lookup.discard(error.uuid)
except KeyError:
continue
err = ErrorObject(
Expand Down
Loading