diff --git a/mock_tests/test_batch.py b/mock_tests/test_batch.py index ad7357f04..459e6b0cf 100644 --- a/mock_tests/test_batch.py +++ b/mock_tests/test_batch.py @@ -1,9 +1,11 @@ +import warnings from typing import AsyncGenerator, Generator, List import grpc import pytest import pytest_asyncio import weaviate +from weaviate.collections.classes.batch import ErrorObject from weaviate.proto.v1 import batch_pb2, weaviate_pb2_grpc from .conftest import MOCK_IP, MOCK_PORT, MOCK_PORT_GRPC, mock_class, HTTPServer @@ -160,3 +162,46 @@ def test_ssb_stream_reports_has_errors( batch.add_object({"name": f"Object {i}"}) assert len(failed_object_stream.batch.failed_objects) == 2 assert failed_object_stream.batch.results.objs.has_errors + + +class MockRateLimitedSecondChunkService(weaviate_pb2_grpc.WeaviateServicer): + calls = 0 + + def BatchObjects(self, request, context): + self.calls += 1 + if self.calls == 2: + return batch_pb2.BatchObjectsReply( + errors=[{"index": 0, "error": "failed with status: 503 error"}] + ) + return batch_pb2.BatchObjectsReply() + + +@pytest.fixture(scope="function") +def rate_limited_readd_collection( + weaviate_mock: HTTPServer, start_grpc_server: grpc.Server +) -> Generator[weaviate.collections.Collection, None, None]: + weaviate_mock.expect_request(f"/v1/schema/{mock_class['class']}").respond_with_json(mock_class) + weaviate_pb2_grpc.add_WeaviateServicer_to_server( + MockRateLimitedSecondChunkService(), start_grpc_server + ) + client = weaviate.connect_to_local(port=MOCK_PORT, host=MOCK_IP, grpc_port=MOCK_PORT_GRPC) + yield client.collections.use(mock_class["class"]) + client.close() + + +def test_readded_rate_limited_object_leaves_no_stale_error( + rate_limited_readd_collection: weaviate.collections.Collection, +) -> None: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with rate_limited_readd_collection.batch.fixed_size( + batch_size=2, concurrent_requests=1 + ) as batch: + for i in range(4): + batch.add_object({"name": f"Object {i}"}) + + results = rate_limited_readd_collection.batch.results.objs + assert results.errors == {} + assert len(results.uuids) == 4 + assert all(not isinstance(resp, ErrorObject) for resp in results._all_responses) + assert not any("Dep020" in str(w.message) for w in caught) diff --git a/weaviate/collections/batch/base.py b/weaviate/collections/batch/base.py index 1d736303c..d0c03e0ee 100644 --- a/weaviate/collections/batch/base.py +++ b/weaviate/collections/batch/base.py @@ -706,19 +706,25 @@ def __send_batch( self.__batch_objects.prepend(readd_objects) + readded_set = set(readded_objects) new_errors = { - i: err for i, err in response_obj.errors.items() if i not in readded_objects + i: err for i, err in response_obj.errors.items() if i not in readded_set + } + # `errors` is keyed by the object's original index while + # `all_responses` is ordered by the object's position in this + # request, so map the re-added objects to positions first. + readded_indices = {response_obj.errors[i].object_.index for i in readded_set} + readded_positions = { + pos for pos, obj in enumerate(objs) if obj.index in readded_indices } response_obj = BatchObjectReturn( - uuids={ - i: uid for i, uid in response_obj.uuids.items() if i not in readded_objects - }, + uuids={i: uid for i, uid in response_obj.uuids.items() if i not in readded_set}, errors=new_errors, has_errors=len(new_errors) > 0, _all_responses=[ - err - for i, err in enumerate(response_obj.all_responses) - if i not in readded_objects + resp + for pos, resp in enumerate(response_obj._all_responses) + if pos not in readded_positions ], elapsed_seconds=response_obj.elapsed_seconds, )