From 17bd0b8dd7a30b63f542ce7d83cefaa3efddcdf5 Mon Sep 17 00:00:00 2001 From: John Trengrove Date: Wed, 11 Nov 2026 12:55:50 +1000 Subject: [PATCH 1/2] Add rescore_factor to quantizer configs and rescore_limit to PQ Exposes the rescoreFactor setting from weaviate/weaviate#13536 on the pq, bq, sq and rq quantizers in Configure, Reconfigure and the config returned by the server. PQ also gains rescore_limit, which the server now honours for every quantizer. Co-Authored-By: Claude --- test/collection/test_config.py | 25 ++++++++++++++++- test/collection/test_config_methods.py | 26 ++++++++++++++++++ test/collection/test_config_update.py | 18 +++++++++++++ weaviate/collections/classes/config.py | 27 ++++++++++++++++--- .../collections/classes/config_methods.py | 5 ++++ .../classes/config_vector_index.py | 22 +++++++++++++++ 6 files changed, 119 insertions(+), 4 deletions(-) diff --git a/test/collection/test_config.py b/test/collection/test_config.py index d793d1487..08691a05f 100644 --- a/test/collection/test_config.py +++ b/test/collection/test_config.py @@ -1,5 +1,5 @@ import warnings -from typing import List, Union +from typing import Any, List, Union import pytest from pydantic import ValidationError @@ -3569,3 +3569,26 @@ def test_reconfigure_inverted_index_without_stopword_presets_leaves_existing(sel } merged = rc.merge_with_existing(existing) assert merged["stopwordPresets"] == {"fr": ["le", "la"]} + + +@pytest.mark.parametrize( + "quantizer,name", + [ + (Configure.VectorIndex.Quantizer.pq(rescore_limit=123, rescore_factor=2.5), "pq"), + (Configure.VectorIndex.Quantizer.bq(rescore_limit=123, rescore_factor=2.5), "bq"), + (Configure.VectorIndex.Quantizer.sq(rescore_limit=123, rescore_factor=2.5), "sq"), + (Configure.VectorIndex.Quantizer.rq(rescore_limit=123, rescore_factor=2.5), "rq"), + ], +) +def test_vector_config_hnsw_quantizer_rescore_factor(quantizer: Any, name: str) -> None: + vi_dict = Configure.VectorIndex.hnsw(quantizer=quantizer)._to_dict() + + assert vi_dict[name]["rescoreLimit"] == 123 + assert vi_dict[name]["rescoreFactor"] == 2.5 + + +def test_vector_config_hnsw_quantizer_rescore_factor_omitted_by_default() -> None: + vi_dict = Configure.VectorIndex.hnsw(quantizer=Configure.VectorIndex.Quantizer.pq())._to_dict() + + assert "rescoreLimit" not in vi_dict["pq"] + assert "rescoreFactor" not in vi_dict["pq"] diff --git a/test/collection/test_config_methods.py b/test/collection/test_config_methods.py index 053199dec..b25810b85 100644 --- a/test/collection/test_config_methods.py +++ b/test/collection/test_config_methods.py @@ -507,3 +507,29 @@ def test_collection_config_stopword_presets_absent() -> None: schema = _full_schema("TestNoStopwordPresets") full = _collection_config_from_json(schema) assert full.inverted_index_config.stopword_presets is None + + +@pytest.mark.parametrize("rescore_factor", [None, -1, 2.5]) +def test_collection_config_from_json_quantizer_rescore_factor( + rescore_factor: Optional[float], +) -> None: + index_config = dict(HFRESH_CONFIG) + if rescore_factor is not None: + index_config["rq"] = {**HFRESH_CONFIG["rq"], "rescoreFactor": rescore_factor} + schema = _schema_with_vector_config( + { + "vec": { + "vectorizer": {"none": {}}, + "vectorIndexType": "hfresh", + "vectorIndexConfig": index_config, + } + } + ) + + config = _collection_config_from_json(schema) + + assert config.vector_config is not None + quantizer = config.vector_config["vec"].vector_index_config.quantizer + assert quantizer is not None + assert quantizer.rescore_limit == 350 + assert quantizer.rescore_factor == rescore_factor diff --git a/test/collection/test_config_update.py b/test/collection/test_config_update.py index 2ed2053c3..c3a5e9703 100644 --- a/test/collection/test_config_update.py +++ b/test/collection/test_config_update.py @@ -365,3 +365,21 @@ def test_updating_vector_when_none_left() -> None: with pytest.raises(WeaviateInvalidInputError, match="does not exist"): update.merge_with_existing({"class": "Test", "properties": []}) + + +def test_updating_rq_rescore_factor() -> None: + schema = _hfresh_schema() + update = _CollectionConfigUpdate( + vector_config=Reconfigure.Vectors.update( + name="boi", + vector_index_config=Reconfigure.VectorIndex.hfresh( + quantizer=Reconfigure.VectorIndex.Quantizer.rq(rescore_factor=3.5) + ), + ) + ) + + new_schema = update.merge_with_existing(schema) + + rq = new_schema["vectorConfig"]["boi"]["vectorIndexConfig"]["rq"] + assert rq["rescoreFactor"] == 3.5 + assert rq["rescoreLimit"] == 20 diff --git a/weaviate/collections/classes/config.py b/weaviate/collections/classes/config.py index 4e20c4692..b42bb6c84 100644 --- a/weaviate/collections/classes/config.py +++ b/weaviate/collections/classes/config.py @@ -2066,6 +2066,8 @@ class _PQConfig(_ConfigBase): centroids: int training_limit: int encoder: PQEncoderConfig + rescore_limit: Optional[int] + rescore_factor: Optional[float] @property def bit_compression(self) -> bool: @@ -2080,11 +2082,13 @@ def bit_compression(self) -> bool: class _BQConfig(_ConfigBase): cache: Optional[bool] rescore_limit: int + rescore_factor: Optional[float] @dataclass class _SQConfig(_ConfigBase): rescore_limit: int + rescore_factor: Optional[float] training_limit: int @@ -2093,6 +2097,7 @@ class _RQConfig(_ConfigBase): cache: Optional[bool] bits: Optional[int] rescore_limit: int + rescore_factor: Optional[float] centering: Optional[bool] training_limit: Optional[int] @@ -3106,6 +3111,8 @@ def pq( segments: Optional[int] = None, training_limit: Optional[int] = None, enabled: bool = True, + rescore_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _PQConfigUpdate: """Create a `_PQConfigUpdate` object to be used when updating the product quantization (PQ) configuration of Weaviate. @@ -3127,10 +3134,16 @@ def pq( if encoder_type is not None or encoder_distribution is not None else None ), + rescoreLimit=rescore_limit, + rescoreFactor=rescore_factor, ) @staticmethod - def bq(rescore_limit: Optional[int] = None, enabled: bool = True) -> _BQConfigUpdate: + def bq( + rescore_limit: Optional[int] = None, + enabled: bool = True, + rescore_factor: Optional[float] = None, + ) -> _BQConfigUpdate: """Create a `_BQConfigUpdate` object to be used when updating the binary quantization (BQ) configuration of Weaviate. Use this method when defining the `quantizer` argument in the `vector_index` configuration in `collection.update()`. @@ -3138,13 +3151,16 @@ def bq(rescore_limit: Optional[int] = None, enabled: bool = True) -> _BQConfigUp Args: See [the docs](https://weaviate.io/developers/weaviate/concepts/vector-index#hnsw-with-compression) for a more detailed view! """ # noqa: D417 (missing argument descriptions in the docstring) - return _BQConfigUpdate(rescoreLimit=rescore_limit, enabled=enabled) + return _BQConfigUpdate( + rescoreLimit=rescore_limit, rescoreFactor=rescore_factor, enabled=enabled + ) @staticmethod def sq( rescore_limit: Optional[int] = None, training_limit: Optional[int] = None, enabled: bool = True, + rescore_factor: Optional[float] = None, ) -> _SQConfigUpdate: """Create a `_SQConfigUpdate` object to be used when updating the scalar quantization (SQ) configuration of Weaviate. @@ -3154,7 +3170,10 @@ def sq( See [the docs](https://weaviate.io/developers/weaviate/concepts/vector-index#hnsw-with-compression) for a more detailed view! """ # noqa: D417 (missing argument descriptions in the docstring) return _SQConfigUpdate( - enabled=enabled, rescoreLimit=rescore_limit, trainingLimit=training_limit + enabled=enabled, + rescoreLimit=rescore_limit, + rescoreFactor=rescore_factor, + trainingLimit=training_limit, ) @staticmethod @@ -3164,6 +3183,7 @@ def rq( bits: Optional[int] = None, centering: Optional[bool] = None, training_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _RQConfigUpdate: """Create a `_RQConfigUpdate` object to be used when updating the Rotational quantization (RQ) configuration of Weaviate. @@ -3175,6 +3195,7 @@ def rq( return _RQConfigUpdate( enabled=enabled, rescoreLimit=rescore_limit, + rescoreFactor=rescore_factor, bits=bits, centering=centering, trainingLimit=training_limit, diff --git a/weaviate/collections/classes/config_methods.py b/weaviate/collections/classes/config_methods.py index 846c06fd5..62d20cc39 100644 --- a/weaviate/collections/classes/config_methods.py +++ b/weaviate/collections/classes/config_methods.py @@ -134,11 +134,13 @@ def __get_quantizer_config( quantizer = _BQConfig( cache=config["bq"].get("cache"), rescore_limit=config["bq"].get("rescoreLimit"), + rescore_factor=config["bq"].get("rescoreFactor"), ) elif "sq" in config and config["sq"]["enabled"]: # values are not present for bq+hnsw quantizer = _SQConfig( rescore_limit=config["sq"].get("rescoreLimit"), + rescore_factor=config["sq"].get("rescoreFactor"), training_limit=config["sq"].get("trainingLimit"), ) elif "pq" in config and config["pq"].get("enabled"): @@ -153,12 +155,15 @@ def __get_quantizer_config( config["pq"].get("encoder", {}).get("distribution") ), ), + rescore_limit=config["pq"].get("rescoreLimit"), + rescore_factor=config["pq"].get("rescoreFactor"), ) elif "rq" in config and config["rq"].get("enabled"): quantizer = _RQConfig( cache=config["rq"].get("cache"), bits=config["rq"].get("bits"), rescore_limit=config["rq"].get("rescoreLimit"), + rescore_factor=config["rq"].get("rescoreFactor"), centering=config["rq"].get("centering"), training_limit=config["rq"].get("trainingLimit"), ) diff --git a/weaviate/collections/classes/config_vector_index.py b/weaviate/collections/classes/config_vector_index.py index 8cb995e1d..682139929 100644 --- a/weaviate/collections/classes/config_vector_index.py +++ b/weaviate/collections/classes/config_vector_index.py @@ -278,6 +278,8 @@ class _PQConfigCreate(_QuantizerConfigCreate): encoder: _PQEncoderConfigCreate segments: Optional[int] trainingLimit: Optional[int] + rescoreLimit: Optional[int] + rescoreFactor: Optional[float] @staticmethod def quantizer_name() -> str: @@ -287,6 +289,7 @@ def quantizer_name() -> str: class _BQConfigCreate(_QuantizerConfigCreate): cache: Optional[bool] rescoreLimit: Optional[int] + rescoreFactor: Optional[float] @staticmethod def quantizer_name() -> str: @@ -295,6 +298,7 @@ def quantizer_name() -> str: class _SQConfigCreate(_QuantizerConfigCreate): rescoreLimit: Optional[int] + rescoreFactor: Optional[float] trainingLimit: Optional[int] @staticmethod @@ -306,6 +310,7 @@ class _RQConfigCreate(_QuantizerConfigCreate): cache: Optional[bool] bits: Optional[int] rescoreLimit: Optional[int] + rescoreFactor: Optional[float] centering: Optional[bool] trainingLimit: Optional[int] @@ -327,6 +332,8 @@ class _PQConfigUpdate(_QuantizerConfigUpdate): segments: Optional[int] trainingLimit: Optional[int] encoder: Optional[_PQEncoderConfigUpdate] + rescoreLimit: Optional[int] + rescoreFactor: Optional[float] @staticmethod def quantizer_name() -> str: @@ -336,6 +343,7 @@ def quantizer_name() -> str: class _BQConfigUpdate(_QuantizerConfigUpdate): enabled: Optional[bool] rescoreLimit: Optional[int] + rescoreFactor: Optional[float] @staticmethod def quantizer_name() -> str: @@ -345,6 +353,7 @@ def quantizer_name() -> str: class _RQConfigUpdate(_QuantizerConfigUpdate): enabled: Optional[bool] rescoreLimit: Optional[int] + rescoreFactor: Optional[float] bits: Optional[int] centering: Optional[bool] trainingLimit: Optional[int] @@ -357,6 +366,7 @@ def quantizer_name() -> str: class _SQConfigUpdate(_QuantizerConfigUpdate): enabled: Optional[bool] rescoreLimit: Optional[int] + rescoreFactor: Optional[float] trainingLimit: Optional[int] @staticmethod @@ -421,6 +431,8 @@ def pq( encoder_type: Optional[PQEncoderType] = None, segments: Optional[int] = None, training_limit: Optional[int] = None, + rescore_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _PQConfigCreate: """Create a `_PQConfigCreate` object to be used when defining the product quantization (PQ) configuration of Weaviate. @@ -436,12 +448,15 @@ def pq( segments=segments, trainingLimit=training_limit, encoder=_PQEncoderConfigCreate(type_=encoder_type, distribution=encoder_distribution), + rescoreLimit=rescore_limit, + rescoreFactor=rescore_factor, ) @staticmethod def bq( cache: Optional[bool] = None, rescore_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _BQConfigCreate: """Create a `_BQConfigCreate` object to be used when defining the binary quantization (BQ) configuration of Weaviate. @@ -453,6 +468,7 @@ def bq( return _BQConfigCreate( cache=cache, rescoreLimit=rescore_limit, + rescoreFactor=rescore_factor, ) @deprecated( @@ -464,6 +480,7 @@ def sq( cache: bool, rescore_limit: Optional[int] = None, training_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _SQConfigCreate: ... @overload @@ -472,6 +489,7 @@ def sq( cache: Literal[None] = None, rescore_limit: Optional[int] = None, training_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _SQConfigCreate: ... @staticmethod @@ -479,6 +497,7 @@ def sq( cache: Optional[bool] = None, rescore_limit: Optional[int] = None, training_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _SQConfigCreate: """Create a `_SQConfigCreate` object to be used when defining the scalar quantization (SQ) configuration of Weaviate. @@ -489,6 +508,7 @@ def sq( """ # noqa: D417 (missing argument descriptions in the docstring) return _SQConfigCreate( rescoreLimit=rescore_limit, + rescoreFactor=rescore_factor, trainingLimit=training_limit, ) @@ -499,6 +519,7 @@ def rq( rescore_limit: Optional[int] = None, centering: Optional[bool] = None, training_limit: Optional[int] = None, + rescore_factor: Optional[float] = None, ) -> _RQConfigCreate: """Create a `_RQConfigCreate` object to be used when defining the Rotational quantization (RQ) configuration of Weaviate. @@ -513,6 +534,7 @@ def rq( rescoreLimit=rescore_limit, centering=centering, trainingLimit=training_limit, + rescoreFactor=rescore_factor, ) @staticmethod From 4de608b0f788ca85be506dd350ee5c3466d2f212 Mon Sep 17 00:00:00 2001 From: John Trengrove Date: Wed, 11 Nov 2026 15:55:53 +1000 Subject: [PATCH 2/2] Add integration test for rescore_factor and run CI against 1.41 Co-Authored-By: Claude --- .github/workflows/main.yaml | 4 ++- integration/test_collection_config.py | 46 ++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 51bd2e25e..a87ea5468 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -32,6 +32,7 @@ env: WEAVIATE_138: 1.38.22 WEAVIATE_139: 1.39.13 WEAVIATE_140: 1.40.2 + WEAVIATE_141: 1.41.0-dev-483b145 jobs: lint-and-format: @@ -387,7 +388,8 @@ jobs: $WEAVIATE_137, $WEAVIATE_138, $WEAVIATE_139, - $WEAVIATE_140 + $WEAVIATE_140, + $WEAVIATE_141 ] steps: - name: Checkout diff --git a/integration/test_collection_config.py b/integration/test_collection_config.py index 70f0af070..653eff291 100644 --- a/integration/test_collection_config.py +++ b/integration/test_collection_config.py @@ -1,6 +1,6 @@ import datetime import time -from typing import Any, Dict, Generator, List, Optional, Union +from typing import Any, Callable, Dict, Generator, List, Optional, Union import pytest as pytest from _pytest.fixtures import SubRequest @@ -706,6 +706,50 @@ def test_hnsw_with_rq4c(collection_factory: CollectionFactory) -> None: assert config.vector_index_config.quantizer.training_limit == 10000 +@pytest.mark.parametrize( + "configure,reconfigure", + [ + (Configure.VectorIndex.Quantizer.pq, Reconfigure.VectorIndex.Quantizer.pq), + (Configure.VectorIndex.Quantizer.bq, Reconfigure.VectorIndex.Quantizer.bq), + (Configure.VectorIndex.Quantizer.sq, Reconfigure.VectorIndex.Quantizer.sq), + (Configure.VectorIndex.Quantizer.rq, Reconfigure.VectorIndex.Quantizer.rq), + ], + ids=["pq", "bq", "sq", "rq"], +) +def test_hnsw_quantizer_rescore_factor( + collection_factory: CollectionFactory, + configure: Callable[..., Any], + reconfigure: Callable[..., Any], +) -> None: + dummy = collection_factory("dummy") + if dummy._connection._weaviate_version.is_lower_than(1, 41, 0): + pytest.skip("rescoreFactor is not supported in Weaviate versions lower than 1.41.0") + + collection = collection_factory( + vector_index_config=Configure.VectorIndex.hnsw( + quantizer=configure(rescore_limit=30, rescore_factor=2.5), + ), + ) + + config = collection.config.get() + assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) + assert config.vector_index_config.quantizer is not None + assert config.vector_index_config.quantizer.rescore_limit == 30 + assert config.vector_index_config.quantizer.rescore_factor == 2.5 + + collection.config.update( + vector_index_config=Reconfigure.VectorIndex.hnsw( + quantizer=reconfigure(rescore_factor=4), + ), + ) + + config = collection.config.get() + assert isinstance(config.vector_index_config, _VectorIndexConfigHNSW) + assert config.vector_index_config.quantizer is not None + assert config.vector_index_config.quantizer.rescore_limit == 30 + assert config.vector_index_config.quantizer.rescore_factor == 4 + + def test_hnsw_with_pathseer_filter_strategy(collection_factory: CollectionFactory) -> None: dummy = collection_factory("dummy") if dummy._connection._weaviate_version.is_lower_than(1, 40, 0):