From 5f605ba9afae7bba700ea1e4ff8e5109bd69886e Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Sun, 26 Jul 2026 06:54:14 -0700 Subject: [PATCH 1/3] fix(index): sort vector search top-k results when not truncating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SearchResult::top_k` only sorted results by score on the truncation path (candidates > k). When `k >= candidate count`, it returned rows in the unordered scored-map / insertion order instead of by relevance. Because the row order was previously discarded downstream (row-range scans re-read in file order), this went unnoticed — but consumers that honor the row order (e.g. the DataFusion `vector_search` scan) then get unranked results whenever `limit >= number of matched rows`. This affects `vector_search`, `full_text_search`, and `hybrid_search`. Sort the no-truncation branch by relevance rank too, and update the unit tests that asserted the old insertion-order behavior. --- crates/paimon/src/vector_search.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/crates/paimon/src/vector_search.rs b/crates/paimon/src/vector_search.rs index 53205cbf..1625243e 100644 --- a/crates/paimon/src/vector_search.rs +++ b/crates/paimon/src/vector_search.rs @@ -218,12 +218,10 @@ impl SearchResult { } if best_by_row_id.len() <= k { - // Keep the original row order when no truncation is needed. - let rows = self - .row_ids - .iter() - .filter_map(|row_id| best_by_row_id.remove(row_id)) - .collect(); + // Still sort best-first: the scored map is unordered, and consumers rely + // on relevance rank. + let mut rows: Vec = best_by_row_id.into_values().collect(); + sort_scored_rows_by_rank(&mut rows); return Self::from_scored_rows(rows); } @@ -382,12 +380,14 @@ mod tests { } #[test] - fn test_search_result_top_k_preserves_order_without_truncation() { + fn test_search_result_top_k_sorts_best_first_without_truncation() { + // Even when k >= candidate count (no truncation), results must be returned + // best-first by score, not in the input/insertion order. let result = SearchResult::new(vec![3, 1, 2], vec![0.1, 0.9, 0.5]); let top = result.top_k(3); - assert_eq!(top.row_ids, result.row_ids); - assert_eq!(top.scores, result.scores); + assert_eq!(top.row_ids, vec![1, 2, 3]); + assert_eq!(top.scores, vec![0.9, 0.5, 0.1]); } #[test] @@ -409,8 +409,9 @@ mod tests { .without_deleted_row_ranges(Some(&deleted)) .unwrap() .top_k(10); - assert_eq!(filtered.row_ids, vec![1, 4]); - assert_eq!(filtered.scores, vec![0.1, 0.2]); + // Rows 2..3 are deleted; the remaining rows come back best-first by score. + assert_eq!(filtered.row_ids, vec![4, 1]); + assert_eq!(filtered.scores, vec![0.2, 0.1]); } #[test] From 97158608e94cb2b7e6b2a6c1ab8b55d4eb198ad2 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Sun, 26 Jul 2026 07:30:10 -0700 Subject: [PATCH 2/3] fix(index): sort full_text_search top-k results when not truncating full_text's SearchResult::top_k had the same bug as the vector one: when `k >= candidate count` it returned rows in input/shard order instead of by relevance score. Drop the no-truncation early-return so both paths sort best-first, and update the test that asserted the old order. --- crates/paimon/src/full_text.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/paimon/src/full_text.rs b/crates/paimon/src/full_text.rs index 0c0b5f71..78ecbdbd 100644 --- a/crates/paimon/src/full_text.rs +++ b/crates/paimon/src/full_text.rs @@ -126,9 +126,8 @@ impl SearchResult { /// Return top-k results by score (descending). pub fn top_k(&self, k: usize) -> Self { - if self.row_ids.len() <= k { - return self.clone(); - } + // Always sort best-first, even when no truncation is needed (len <= k): + // downstream consumers rely on relevance order, not input/shard order. let mut indices: Vec = (0..self.row_ids.len()).collect(); indices.sort_by(|&a, &b| { self.scores[b] @@ -258,8 +257,20 @@ mod tests { .without_deleted_row_ranges(Some(&deleted)) .unwrap() .top_k(10); - assert_eq!(filtered.row_ids, vec![1, 4]); - assert_eq!(filtered.scores, vec![0.1, 0.2]); + // Rows 2..3 are deleted; the remaining rows come back best-first by score. + assert_eq!(filtered.row_ids, vec![4, 1]); + assert_eq!(filtered.scores, vec![0.2, 0.1]); + } + + #[test] + fn test_search_result_top_k_sorts_best_first_without_truncation() { + // k >= candidate count: results must still be best-first by score, not in + // input/shard order. + let result = SearchResult::new(vec![3, 1, 2], vec![0.1, 0.9, 0.5]); + + let top = result.top_k(3); + assert_eq!(top.row_ids, vec![1, 2, 3]); + assert_eq!(top.scores, vec![0.9, 0.5, 0.1]); } #[test] From 7191e103802a8be53f361f7a978d4d2476e79575 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Sun, 26 Jul 2026 07:43:18 -0700 Subject: [PATCH 3/3] fix(index): make full_text top-k comparator match vector search Use total_cmp (NaN-safe) and an explicit smaller-row-id tie-break in full_text's top_k, matching vector_search's ordering. This keeps the result deterministic regardless of input/shard order rather than relying on a stable sort over the input sequence. --- crates/paimon/src/full_text.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/paimon/src/full_text.rs b/crates/paimon/src/full_text.rs index 78ecbdbd..84aee970 100644 --- a/crates/paimon/src/full_text.rs +++ b/crates/paimon/src/full_text.rs @@ -129,10 +129,13 @@ impl SearchResult { // Always sort best-first, even when no truncation is needed (len <= k): // downstream consumers rely on relevance order, not input/shard order. let mut indices: Vec = (0..self.row_ids.len()).collect(); + // Best-first by score (total_cmp is NaN-safe), then smaller row id. This + // matches vector_search's top_k and keeps the order deterministic regardless + // of the input/shard order. indices.sort_by(|&a, &b| { self.scores[b] - .partial_cmp(&self.scores[a]) - .unwrap_or(std::cmp::Ordering::Equal) + .total_cmp(&self.scores[a]) + .then_with(|| self.row_ids[a].cmp(&self.row_ids[b])) }); indices.truncate(k); let row_ids = indices.iter().map(|&i| self.row_ids[i]).collect(); @@ -273,6 +276,16 @@ mod tests { assert_eq!(top.scores, vec![0.9, 0.5, 0.1]); } + #[test] + fn test_search_result_top_k_tie_breaks_by_smaller_row_id() { + // Equal scores: order by smaller row id, independent of input order. + let result = SearchResult::new(vec![30, 10, 20], vec![0.9, 0.9, 0.9]); + + let top = result.top_k(3); + assert_eq!(top.row_ids, vec![10, 20, 30]); + assert_eq!(top.scores, vec![0.9, 0.9, 0.9]); + } + #[test] fn test_search_result_or_preserves_left_score_and_order() { let left = SearchResult::new(vec![1, 2], vec![0.4, 0.5]);