From c6def699d55e94136dbb51bfdfd58559bf16da99 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 3 Jul 2026 16:35:09 +0200 Subject: [PATCH 1/5] Implement a proper metric variant --- crates/geo_filters/Cargo.toml | 5 + crates/geo_filters/README.md | 55 +++ .../evaluation/nearest_neighbor.rs | 128 +++++++ crates/geo_filters/src/diff_count.rs | 2 + crates/geo_filters/src/diff_count/bitvec.rs | 13 + crates/geo_filters/src/diff_count/config.rs | 2 +- crates/geo_filters/src/diff_count/metric.rs | 319 ++++++++++++++++++ 7 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 crates/geo_filters/evaluation/nearest_neighbor.rs create mode 100644 crates/geo_filters/src/diff_count/metric.rs diff --git a/crates/geo_filters/Cargo.toml b/crates/geo_filters/Cargo.toml index 62670385..ef55a2e6 100644 --- a/crates/geo_filters/Cargo.toml +++ b/crates/geo_filters/Cargo.toml @@ -49,6 +49,11 @@ path = "evaluation/performance.rs" harness = false required-features = ["evaluation"] +[[bench]] +name = "nearest_neighbor" +path = "evaluation/nearest_neighbor.rs" +harness = false + [[bin]] name = "accuracy" path = "evaluation/accuracy.rs" diff --git a/crates/geo_filters/README.md b/crates/geo_filters/README.md index 8b5ebf2f..18ed6ff1 100644 --- a/crates/geo_filters/README.md +++ b/crates/geo_filters/README.md @@ -61,6 +61,61 @@ Our implementation is at least as fast as the HLL++ port for all sizes. The `GeoDistinctCount` solution only needs to count the number of occupied buckets and the offset of the first bucket. Both numbers are tracked during incremental updates, such that the distinct count can be computed in constant time. +## Nearest-neighbor similarity metric + +`GeoDiffCount` also doubles as a compact similarity sketch. The number of differing one-bits between +two filters — the Hamming distance of their bit representations — is a simple, uncalibrated distance +that grows with the true symmetric-difference size. This is useful for nearest-neighbor search, e.g. +to find the most similar repository or document. + +[`GeoDiffMetric`](https://docs.rs/geo_filters/latest/geo_filters/diff_count/struct.GeoDiffMetric.html) +wraps a `GeoDiffCount`, caches its one-bit count, and implements the `MetricSpace` trait: + +- `size()` returns the filter's one-bit count as a `Metric` value; +- `symmetric_diff_size(other, bound)` returns the exact one-bit distance to another filter, but + abandons the computation once it reaches `bound` (returning an "infinite" value), so far-away + candidates are rejected cheaply while scanning a candidate list; +- `size().abs_diff(&other.size())` is an `O(1)` reverse-triangle lower bound on that distance. + +```rust +use geo_filters::diff_count::{GeoDiffCount7, GeoDiffMetric, OnesMetric}; +use geo_filters::{Count, Metric, MetricSpace}; + +let mut a = GeoDiffCount7::default(); +(0..1000u64).for_each(|i| a.push(i)); +let mut b = GeoDiffCount7::default(); +(500..1500u64).for_each(|i| b.push(i)); + +let a = GeoDiffMetric::new(a); +let b = GeoDiffMetric::new(b); + +// O(1) lower bound from the cached sizes, then the exact distance (unbounded). +let lower_bound = a.size().abs_diff(&b.size()); +let distance = a.symmetric_diff_size(&b, OnesMetric::infinite()); +assert!(lower_bound <= distance); +``` + +Time to compare a query filter (1M items) with a candidate, given the distance to a known near +neighbor as the pruning bound. A `far` candidate is disjoint (farther than the bound, so it is +rejected), while a `near` candidate is a closer near-duplicate (below the bound, so it becomes the +new best and is scanned in full). Release build; absolute numbers are hardware-dependent: + +| configuration | candidate | `estimate` | `symmetric_diff_size` | `symmetric_diff_size` (capped) | +| ----------------- | --------- | ---------- | --------------------- | ------------------------------ | +| `GeoDiffConfig7` | far | 115 ns | 60 ns | 12 ns | +| `GeoDiffConfig7` | near | 149 ns | 50 ns | 51 ns | +| `GeoDiffConfig10` | far | 633 ns | 374 ns | 74 ns | +| `GeoDiffConfig10` | near | 918 ns | 298 ns | 300 ns | +| `GeoDiffConfig13` | far | 4.69 µs | 2.44 µs | 337 ns | +| `GeoDiffConfig13` | near | 6.92 µs | 1.90 µs | 1.92 µs | + +`estimate` is the calibrated `size_with_sketch` estimate and `symmetric_diff_size` the exact bit +distance; the capped variant abandons once the running distance reaches the bound. For a `far` +candidate it abandons almost immediately (~7× faster than the exact distance, ~14× faster than the +estimate), whereas a `near` candidate is below the bound and therefore scanned in full — the capped +and exact costs match. Since a nearest-neighbor scan rejects far more candidates than it keeps, the +early abandon dominates the search cost. Reproduce with `cargo bench --bench nearest_neighbor`. + ## Evaluation Accuracy and performance evaluations for the predefined filter configurations are included in the repository: diff --git a/crates/geo_filters/evaluation/nearest_neighbor.rs b/crates/geo_filters/evaluation/nearest_neighbor.rs new file mode 100644 index 00000000..a2c81511 --- /dev/null +++ b/crates/geo_filters/evaluation/nearest_neighbor.rs @@ -0,0 +1,128 @@ +//! Benchmarks the different ways to measure (dis)similarity of `GeoDiffCount` filters, in a +//! nearest-neighbor setting where a `query` is compared against many candidates. +//! +//! Two families are compared: +//! +//! * The calibrated *size estimate* ([`Count::size`] / [`Count::size_with_sketch`]), which scans +//! only a bounded window and upscales. +//! * The exact *bit count* via [`GeoDiffMetric`], a simple, uncalibrated metric based on the number +//! of differing one-bits (Hamming distance). It caches each filter's one-bit count (`size`) and +//! exposes `symmetric_diff_size`, the exact distance, abandoned once a given bound is reached. +//! +//! Both are measured for a `far` candidate (disjoint, rejected by the bound) and a `kept` candidate +//! (a near-duplicate, roughly at the bound and scanned in full), to contrast pruning vs. keeping. +//! +//! Groups: +//! * `size:*` - single filter: `estimate` (calibrated) vs. `metric` (exact one-bit count). +//! * `diff:*` - pairwise: `estimate`/`symmetric_diff_size`/`symmetric_diff_size_capped` for the +//! `far` and `kept` candidates. + +use std::hint::black_box; + +use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; +use geo_filters::config::GeoConfig; +use geo_filters::diff_count::{ + GeoDiffConfig10, GeoDiffConfig13, GeoDiffConfig7, GeoDiffCount, GeoDiffMetric, OnesMetric, +}; +use geo_filters::{Count, Diff, Metric, MetricSpace}; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha12Rng; + +fn build + Default>( + rng: &mut ChaCha12Rng, + n: usize, +) -> GeoDiffCount<'static, C> { + let mut f = GeoDiffCount::::default(); + for _ in 0..n { + f.push_hash(rng.next_u64()); + } + f +} + +/// Returns a near-duplicate of `base` that differs from it by roughly `diff` items. +fn near + Default>( + rng: &mut ChaCha12Rng, + base: &GeoDiffCount<'static, C>, + diff: usize, +) -> GeoDiffCount<'static, C> { + let mut f = base.clone(); + for _ in 0..diff { + f.push_hash(rng.next_u64()); + } + f +} + +fn bench_config + Default>(c: &mut Criterion, name: &str, items: usize) { + let mut rng = ChaCha12Rng::seed_from_u64(42); + + // Each filter is owned by its metric, which caches the one-bit count, as a real index would. + let query_m = GeoDiffMetric::new(build::(&mut rng, items)); + // A far-away candidate (disjoint set), and a near neighbor differing by ~10% of the base filter + // whose distance provides the prior bound. + let far_m = GeoDiffMetric::new(build::(&mut rng, items)); + let bound_m = GeoDiffMetric::new(near(&mut rng, query_m.filter(), (items / 10).max(1))); + // A kept candidate: a closer ~5% near-duplicate, i.e. below the bound and hence scanned in full. + let kept_m = GeoDiffMetric::new(near(&mut rng, query_m.filter(), (items / 20).max(1))); + let ones_bound = query_m.symmetric_diff_size(&bound_m, OnesMetric::infinite()); + + // Single-filter size metric: the estimate vs. the exact bit count computed when constructing a + // `GeoDiffMetric` (the filter is cloned in the untimed setup so only the count is measured). + let mut group = c.benchmark_group(format!("size:{name}/{items}")); + group.bench_function("estimate", |b| { + b.iter(|| black_box(query_m.filter().size())) + }); + group.bench_function("metric", |b| { + b.iter_batched( + || query_m.filter().clone(), + |f| black_box(GeoDiffMetric::new(f).size()), + BatchSize::SmallInput, + ) + }); + group.finish(); + + // Diff between two filters, for a `far` candidate (disjoint, pruned by the bound) and a `kept` + // candidate (a ~10% near-duplicate, roughly at the bound and scanned in full): + // * `estimate`: calibrated size of the symmetric difference (`Count::size_with_sketch`); + // * `symmetric_diff_size`: exact one-bit distance (`MetricSpace::symmetric_diff_size` with an + // infinite bound), scanning both filters in full; + // * `symmetric_diff_size_capped`: same, but abandons once `ones_bound` differing bits are reached. + let mut group = c.benchmark_group(format!("diff:{name}/{items}")); + group.bench_function("estimate_far", |b| { + b.iter(|| black_box(query_m.filter().size_with_sketch(black_box(far_m.filter())))) + }); + group.bench_function("symmetric_diff_size_far", |b| { + b.iter(|| black_box(query_m.symmetric_diff_size(black_box(&far_m), OnesMetric::infinite()))) + }); + group.bench_function("symmetric_diff_size_capped_far", |b| { + b.iter(|| black_box(query_m.symmetric_diff_size(black_box(&far_m), ones_bound))) + }); + group.bench_function("estimate_kept", |b| { + b.iter(|| { + black_box( + query_m + .filter() + .size_with_sketch(black_box(kept_m.filter())), + ) + }) + }); + group.bench_function("symmetric_diff_size_kept", |b| { + b.iter(|| { + black_box(query_m.symmetric_diff_size(black_box(&kept_m), OnesMetric::infinite())) + }) + }); + group.bench_function("symmetric_diff_size_capped_kept", |b| { + b.iter(|| black_box(query_m.symmetric_diff_size(black_box(&kept_m), ones_bound))) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + for items in [1_000, 10_000, 100_000, 1_000_000] { + bench_config::(c, "config7", items); + bench_config::(c, "config10", items); + bench_config::(c, "config13", items); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/geo_filters/src/diff_count.rs b/crates/geo_filters/src/diff_count.rs index 728cd1f3..44f1f088 100644 --- a/crates/geo_filters/src/diff_count.rs +++ b/crates/geo_filters/src/diff_count.rs @@ -14,10 +14,12 @@ use crate::{Count, Diff}; mod bitvec; mod config; +mod metric; mod sim_hash; use bitvec::*; pub use config::{GeoDiffConfig10, GeoDiffConfig13, GeoDiffConfig7}; +pub use metric::{GeoDiffMetric, OnesMetric}; pub use sim_hash::SimHash; /// Diff count filter with a relative error standard deviation of ~0.125. diff --git a/crates/geo_filters/src/diff_count/bitvec.rs b/crates/geo_filters/src/diff_count/bitvec.rs index f77323c2..a9e367a0 100644 --- a/crates/geo_filters/src/diff_count/bitvec.rs +++ b/crates/geo_filters/src/diff_count/bitvec.rs @@ -72,6 +72,19 @@ impl BitVec<'_> { self.num_bits } + /// The total number of one-bits set in the vector. + pub fn count_ones(&self) -> usize { + self.blocks + .iter() + .map(|block| block.count_ones() as usize) + .sum() + } + + /// The raw 64-bit blocks backing the vector, from least to most significant. + pub fn blocks(&self) -> &[u64] { + self.blocks.deref() + } + pub fn is_empty(&self) -> bool { self.num_bits() == 0 } diff --git a/crates/geo_filters/src/diff_count/config.rs b/crates/geo_filters/src/diff_count/config.rs index 689004b0..89a6c238 100644 --- a/crates/geo_filters/src/diff_count/config.rs +++ b/crates/geo_filters/src/diff_count/config.rs @@ -63,7 +63,7 @@ impl Lookups for Diff { // and the loop be aborted. // Note: Increasing the constant to switch earlier into approximation mode will mostly increase the // approximation error, but not reduce the runtime of the function by much. -fn expected_diff_buckets(phi: f64, items: f64) -> (f64, f64) { +pub(crate) fn expected_diff_buckets(phi: f64, items: f64) -> (f64, f64) { let mut sum = 0.0; let mut derivative = 0.0; for k in 0.. { diff --git a/crates/geo_filters/src/diff_count/metric.rs b/crates/geo_filters/src/diff_count/metric.rs new file mode 100644 index 00000000..90188468 --- /dev/null +++ b/crates/geo_filters/src/diff_count/metric.rs @@ -0,0 +1,319 @@ +//! A simple, uncalibrated similarity metric for [`GeoDiffCount`] filters based on the exact number +//! of differing one-bits (the Hamming distance between their bit representations). + +use std::cmp::Ordering; +use std::fmt; +use std::ops::Add; + +use super::config::expected_diff_buckets; +use crate::config::{ + estimate_count, xor_bit_chunks, BitChunk, GeoConfig, IsBucketType, BITS_PER_BLOCK, +}; +use crate::diff_count::GeoDiffCount; +use crate::{Diff, Metric, MetricSpace}; + +/// A metric value measured in one-bits: the number of set bits of a filter, or the Hamming distance +/// between two filters. +/// +/// The value is stored in the same (small) integer type `C::BucketType` used for the filter's +/// most-significant bucket positions, and is tagged with the configuration `C`, so metrics of +/// differently configured filters cannot be mixed. The largest representable value doubles as the +/// [`Metric::infinite`] sentinel; construction saturates into it. +pub struct OnesMetric>(C::BucketType); + +impl> OnesMetric { + /// Wraps a raw one-bit count, saturating at [`Metric::infinite`]. + fn new(count: usize) -> Self { + let max = C::BucketType::from_usize(usize::MAX).into_usize(); + Self(C::BucketType::from_usize(count.min(max))) + } + + /// The wrapped one-bit count. + fn get(self) -> usize { + self.0.into_usize() + } +} + +impl + Default> Metric for OnesMetric { + fn zero() -> Self { + Self(C::BucketType::from_usize(0)) + } + + fn infinite() -> Self { + Self(C::BucketType::from_usize(usize::MAX)) + } + + fn abs_diff(&self, other: &Self) -> Self { + Self::new(self.get().abs_diff(other.get())) + } + + /// The calibrated size (item count) that this number of one-bits represents, obtained by + /// inverting the expected-buckets function with the same Newton iteration used to build the + /// estimation lookup table. + fn as_f32(&self) -> f32 { + if *self == Self::infinite() { + return f32::INFINITY; + } + let phi = C::default().phi_f64(); + estimate_count(phi, self.get() as f64, expected_diff_buckets).0 as f32 + } + + /// The number of one-bits expected for the given calibrated size, obtained by evaluating the + /// forward expected-buckets function. + fn from_f32(value: f32) -> Self { + if !value.is_finite() { + return Self::infinite(); + } + let phi = C::default().phi_f64(); + let buckets = expected_diff_buckets(phi, value.max(0.0) as f64).0; + Self::new(buckets.round() as usize) + } +} + +// `Add` saturates into `infinite`, so it is a total operation. +impl> Add for OnesMetric { + type Output = Self; + fn add(self, rhs: Self) -> Self { + Self::new(self.get() + rhs.get()) + } +} + +// The following trait implementations are written by hand rather than derived so that they do not +// impose the corresponding bounds on the phantom configuration type `C`. +impl> Clone for OnesMetric { + fn clone(&self) -> Self { + *self + } +} + +impl> Copy for OnesMetric {} + +impl> PartialEq for OnesMetric { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +impl> Eq for OnesMetric {} + +impl> PartialOrd for OnesMetric { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl> Ord for OnesMetric { + fn cmp(&self, other: &Self) -> Ordering { + self.0.cmp(&other.0) + } +} + +impl> fmt::Debug for OnesMetric { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "OnesMetric({})", self.get()) + } +} + +/// A [`GeoDiffCount`] paired with its cached one-bit count. +/// +/// Taking ownership of a filter and caching its number of one-bits lets the exact bit-count +/// ("ones") distance to other filters be computed repeatedly - as needed for nearest-neighbor +/// search - and, given a bound, abandoned early. The cached count (see [`Self::size`]) also yields +/// an O(1) reverse-triangle lower bound on the distance. Once the nearest neighbor is found, +/// [`Self::filter`] gives access to the underlying filter for a calibrated size estimate. +pub struct GeoDiffMetric<'a, C: GeoConfig> { + filter: GeoDiffCount<'a, C>, + ones: OnesMetric, +} + +impl<'a, C: GeoConfig> GeoDiffMetric<'a, C> { + /// Takes ownership of `filter`, caching its number of one-bits. + pub fn new(filter: GeoDiffCount<'a, C>) -> Self { + let ones = OnesMetric::new(filter.msb.len() + filter.lsb.count_ones()); + Self { filter, ones } + } + + /// The wrapped filter, e.g. to compute a calibrated size estimate once the nearest neighbor is + /// known. + pub fn filter(&self) -> &GeoDiffCount<'a, C> { + &self.filter + } +} + +impl + Default> MetricSpace for GeoDiffMetric<'_, C> { + type Metric = OnesMetric; + + /// The size of the wrapped filter, measured as its number of one-bits. The reverse-triangle + /// lower bound on the distance between two filters is `a.size().abs_diff(&b.size())`. + fn size(&self) -> OnesMetric { + self.ones + } + + /// The exact ones-distance (Hamming distance) to `other`, but abandoned as soon as at least + /// `bound` differing bits have been counted, in which case [`Metric::infinite`] is returned. + /// Otherwise (the distance is strictly below `bound`) the exact distance is returned. Pass + /// [`Metric::infinite`] as the bound to always compute the exact distance. + /// + /// A reverse-triangle rejection using the cached one-bit counts (`|ones(a) - ones(b)| <= + /// distance(a, b)`) discards far candidates in O(1). Otherwise the dense lower bits shared by + /// both filters are XOR-counted block by block directly, which is much faster than the general + /// bit-chunk merge; only the sparse upper region (the boundary block and everything above it, + /// where the most-significant positions live) falls back to the merge. Counting from least to + /// most significant reaches `bound` early for far-apart filters, whose differing bits are + /// concentrated in the dense lower region. + fn symmetric_diff_size(&self, other: &Self, bound: OnesMetric) -> OnesMetric { + let (a, b) = (&self.filter, &other.filter); + assert!( + a.config == b.config, + "combined filters must have the same configuration" + ); + + // O(1) reverse-triangle rejection using the cached one-bit counts. + if self.ones.abs_diff(&other.ones) >= bound { + return OnesMetric::infinite(); + } + let bound = bound.get(); + + // Complete lower blocks that are dense (least-significant bits only, no most-significant + // positions) in both filters, and can therefore be xor-ed directly. + let common_full = a.lsb.num_bits().min(b.lsb.num_bits()) / BITS_PER_BLOCK; + + let mut count = 0; + let (a_blocks, b_blocks) = (a.lsb.blocks(), b.lsb.blocks()); + for i in 0..common_full { + count += (a_blocks[i] ^ b_blocks[i]).count_ones() as usize; + if count >= bound { + return OnesMetric::infinite(); + } + } + + // Sparse upper region: the boundary block and everything above it, where the filters are no + // longer both dense and most-significant positions appear. + for BitChunk { block, .. } in xor_bit_chunks(a.bit_chunks(), b.bit_chunks()) + .take_while(|BitChunk { index, .. }| *index >= common_full) + { + count += block.count_ones() as usize; + if count >= bound { + return OnesMetric::infinite(); + } + } + OnesMetric::new(count) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::GeoConfig; + use crate::diff_count::{xor, GeoDiffConfig7, GeoDiffCount7}; + use crate::{Count, Metric, MetricSpace}; + + fn mix(x: u64) -> u64 { + x.wrapping_add(1).wrapping_mul(0x9E37_79B9_7F4A_7C15) + } + + /// Builds a filter with `shared` hashes (indices `0..shared`) plus `extra` hashes offset by + /// `extra_offset` so that two builds share only their common prefix. + fn build(shared: usize, extra_offset: u64, extra: usize) -> GeoDiffCount7<'static> { + let mut f = GeoDiffCount7::default(); + for i in 0..shared as u64 { + f.push_hash(mix(i)); + } + for i in 0..extra as u64 { + f.push_hash(mix(extra_offset + i)); + } + f + } + + #[test] + fn test_ones_metric() { + for &(shared, a_only, b_only) in &[ + (0usize, 1usize, 1usize), + (0, 200, 150), + (5000, 5000, 5000), + (100000, 100, 100), + ] { + let a = build(shared, 1 << 40, a_only); + let b = build(shared, 1 << 41, b_only); + + let a_ones = a.iter_ones().count(); + let b_ones = b.iter_ones().count(); + let hamming = xor(&a, &b).iter_ones().count(); + + let am = GeoDiffMetric::new(a); + let bm = GeoDiffMetric::new(b); + + // Cached `size` (one-bit count) equals the exact number of set bits. + assert_eq!(am.size(), OnesMetric::new(a_ones)); + assert_eq!(bm.size(), OnesMetric::new(b_ones)); + + // The exact (uncapped) distance is the Hamming distance and is symmetric. + let inf = OnesMetric::infinite(); + let d = am.symmetric_diff_size(&bm, inf); + assert_eq!(d, OnesMetric::new(hamming)); + assert_eq!(d, bm.symmetric_diff_size(&am, inf)); + + // Reverse-triangle lower bound, computed from the cached sizes: symmetric, `<= distance`. + let lb = am.size().abs_diff(&bm.size()); + assert_eq!(lb, bm.size().abs_diff(&am.size())); + assert!(lb <= d); + + // Capped distance: exact strictly below the bound, `infinite()` once reached. + assert!(hamming > 0, "test cases have a non-empty diff"); + for &bound in &[hamming / 2, hamming, hamming + 1, hamming * 2 + 1] { + let capped = am.symmetric_diff_size(&bm, OnesMetric::new(bound)); + if hamming < bound { + assert_eq!(capped, d, "exact below bound {bound}"); + } else { + assert_eq!( + capped, + OnesMetric::infinite(), + "infinite at/above bound {bound}" + ); + } + // Symmetric in its arguments. + assert_eq!(capped, bm.symmetric_diff_size(&am, OnesMetric::new(bound))); + } + + // The lower bound and zero also trigger the O(1) rejection. + assert_eq!(am.symmetric_diff_size(&bm, lb), OnesMetric::infinite()); + assert_eq!( + am.symmetric_diff_size(&bm, OnesMetric::zero()), + OnesMetric::infinite() + ); + } + } + + #[test] + fn test_ones_metric_f32() { + type M = OnesMetric; + + // `zero` and `infinite` map to the expected floating-point values, both ways. + assert_eq!(M::zero().as_f32(), 0.0); + assert_eq!(M::infinite().as_f32(), f32::INFINITY); + assert_eq!(M::from_f32(f32::INFINITY), M::infinite()); + assert_eq!(M::from_f32(f32::NAN), M::infinite()); + assert_eq!(M::from_f32(0.0), M::zero()); + assert_eq!(M::from_f32(-5.0), M::zero()); + + // `as_f32` (Newton inverse) matches the LUT-based calibrated estimate, and `from_f32` (the + // forward function) round-trips the bucket count. + let config: GeoDiffConfig7 = Default::default(); + for &buckets in &[1usize, 5, 20, 50, 100, 300] { + let f = M::new(buckets).as_f32(); + let lut = config.expected_items(buckets); + assert!( + (f - lut).abs() <= 0.02 * lut.max(1.0) + 0.5, + "as_f32 {f} should match the LUT estimate {lut} for {buckets} buckets" + ); + let round_trip = M::from_f32(f).get(); + assert!( + round_trip.abs_diff(buckets) <= 1, + "from_f32 round-trip {round_trip} should recover {buckets} buckets" + ); + } + + // `as_f32` is monotonically increasing in the bucket count. + assert!(M::new(10).as_f32() < M::new(100).as_f32()); + } +} From c4a8421bfbe4156ee8aa1e272e191b105377f316 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 3 Jul 2026 16:35:56 +0200 Subject: [PATCH 2/5] Update lib.rs --- crates/geo_filters/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/geo_filters/src/lib.rs b/crates/geo_filters/src/lib.rs index 29674547..94aa5457 100644 --- a/crates/geo_filters/src/lib.rs +++ b/crates/geo_filters/src/lib.rs @@ -17,6 +17,7 @@ pub mod evaluation; mod test_rng; use std::hash::Hash; +use std::ops::Add; /// Marker trait to indicate the variant implemented by a [`Count`] instance. pub trait Method: Clone + Eq + PartialEq + Send + Sync {} @@ -31,6 +32,39 @@ impl Method for Diff {} pub struct Distinct {} impl Method for Distinct {} +/// A comparable distance value with the few operations needed by nearest-neighbor search: addition, +/// the zero distance, an "unreachable"/infinite value, the absolute difference of two values, and +/// conversion to and from a floating-point value (the calibrated size the metric represents). +pub trait Metric: Ord + Add + Sized { + /// The zero distance. + fn zero() -> Self; + /// An unreachable distance, larger than any real one. + fn infinite() -> Self; + /// The absolute difference between two distances. + fn abs_diff(&self, other: &Self) -> Self; + /// The metric as a floating-point value. + fn as_f32(&self) -> f32; + /// The metric closest to the given floating-point value. + fn from_f32(value: f32) -> Self; +} + +/// A type that can be measured (its [`size`](MetricSpace::size)) and compared to another value of +/// the same type (their [`symmetric_diff_size`](MetricSpace::symmetric_diff_size)), both yielding a value of +/// the associated [`Metric`] type. This abstracts the filter-based similarity metrics from the +/// nearest-neighbor search that consumes them. +pub trait MetricSpace { + /// The metric type produced by [`Self::size`] and [`Self::symmetric_diff_size`]. + type Metric: Metric; + + /// The size of this value. + fn size(&self) -> Self::Metric; + + /// The symmetric difference between this value and `other`, abandoned once it reaches `bound` + /// (in which case a value `>= bound`, e.g. [`Metric::infinite`], is returned). Pass + /// [`Metric::infinite`] to always compute the exact value. + fn symmetric_diff_size(&self, other: &Self, bound: Self::Metric) -> Self::Metric; +} + /// Trait for types solving the set cardinality estimation problem. pub trait Count { /// Add the given hash to the set. From 19cf40f915952e7de20de67b7483d81311f0b4e2 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 6 Jul 2026 08:15:05 +0200 Subject: [PATCH 3/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- crates/geo_filters/src/diff_count/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/geo_filters/src/diff_count/config.rs b/crates/geo_filters/src/diff_count/config.rs index 89a6c238..b34b2aa9 100644 --- a/crates/geo_filters/src/diff_count/config.rs +++ b/crates/geo_filters/src/diff_count/config.rs @@ -63,7 +63,7 @@ impl Lookups for Diff { // and the loop be aborted. // Note: Increasing the constant to switch earlier into approximation mode will mostly increase the // approximation error, but not reduce the runtime of the function by much. -pub(crate) fn expected_diff_buckets(phi: f64, items: f64) -> (f64, f64) { +pub(super) fn expected_diff_buckets(phi: f64, items: f64) -> (f64, f64) { let mut sum = 0.0; let mut derivative = 0.0; for k in 0.. { From 4bee3bc71fbd37a909fc79a42447293826a528e9 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 6 Jul 2026 08:15:14 +0200 Subject: [PATCH 4/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- crates/geo_filters/evaluation/nearest_neighbor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/geo_filters/evaluation/nearest_neighbor.rs b/crates/geo_filters/evaluation/nearest_neighbor.rs index a2c81511..2092ca4c 100644 --- a/crates/geo_filters/evaluation/nearest_neighbor.rs +++ b/crates/geo_filters/evaluation/nearest_neighbor.rs @@ -57,7 +57,7 @@ fn bench_config + Default>(c: &mut Criterion, name: &str, ite // Each filter is owned by its metric, which caches the one-bit count, as a real index would. let query_m = GeoDiffMetric::new(build::(&mut rng, items)); - // A far-away candidate (disjoint set), and a near neighbor differing by ~10% of the base filter + // A far-away candidate (an independent random set), and a near neighbor differing by ~10% of the base filter // whose distance provides the prior bound. let far_m = GeoDiffMetric::new(build::(&mut rng, items)); let bound_m = GeoDiffMetric::new(near(&mut rng, query_m.filter(), (items / 10).max(1))); From e11f6dc66cd5173d0c65f3d8ef620135ee9435eb Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 6 Jul 2026 12:11:32 +0200 Subject: [PATCH 5/5] rename as_f32 -> to_f32 --- crates/geo_filters/src/diff_count/metric.rs | 16 ++++++++-------- crates/geo_filters/src/lib.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/geo_filters/src/diff_count/metric.rs b/crates/geo_filters/src/diff_count/metric.rs index 90188468..aa70eb27 100644 --- a/crates/geo_filters/src/diff_count/metric.rs +++ b/crates/geo_filters/src/diff_count/metric.rs @@ -50,7 +50,7 @@ impl + Default> Metric for OnesMetric { /// The calibrated size (item count) that this number of one-bits represents, obtained by /// inverting the expected-buckets function with the same Newton iteration used to build the /// estimation lookup table. - fn as_f32(&self) -> f32 { + fn to_f32(&self) -> f32 { if *self == Self::infinite() { return f32::INFINITY; } @@ -289,22 +289,22 @@ mod tests { type M = OnesMetric; // `zero` and `infinite` map to the expected floating-point values, both ways. - assert_eq!(M::zero().as_f32(), 0.0); - assert_eq!(M::infinite().as_f32(), f32::INFINITY); + assert_eq!(M::zero().to_f32(), 0.0); + assert_eq!(M::infinite().to_f32(), f32::INFINITY); assert_eq!(M::from_f32(f32::INFINITY), M::infinite()); assert_eq!(M::from_f32(f32::NAN), M::infinite()); assert_eq!(M::from_f32(0.0), M::zero()); assert_eq!(M::from_f32(-5.0), M::zero()); - // `as_f32` (Newton inverse) matches the LUT-based calibrated estimate, and `from_f32` (the + // `to_f32` (Newton inverse) matches the LUT-based calibrated estimate, and `from_f32` (the // forward function) round-trips the bucket count. let config: GeoDiffConfig7 = Default::default(); for &buckets in &[1usize, 5, 20, 50, 100, 300] { - let f = M::new(buckets).as_f32(); + let f = M::new(buckets).to_f32(); let lut = config.expected_items(buckets); assert!( (f - lut).abs() <= 0.02 * lut.max(1.0) + 0.5, - "as_f32 {f} should match the LUT estimate {lut} for {buckets} buckets" + "to_f32 {f} should match the LUT estimate {lut} for {buckets} buckets" ); let round_trip = M::from_f32(f).get(); assert!( @@ -313,7 +313,7 @@ mod tests { ); } - // `as_f32` is monotonically increasing in the bucket count. - assert!(M::new(10).as_f32() < M::new(100).as_f32()); + // `to_f32` is monotonically increasing in the bucket count. + assert!(M::new(10).to_f32() < M::new(100).to_f32()); } } diff --git a/crates/geo_filters/src/lib.rs b/crates/geo_filters/src/lib.rs index 94aa5457..7d29b284 100644 --- a/crates/geo_filters/src/lib.rs +++ b/crates/geo_filters/src/lib.rs @@ -43,7 +43,7 @@ pub trait Metric: Ord + Add + Sized { /// The absolute difference between two distances. fn abs_diff(&self, other: &Self) -> Self; /// The metric as a floating-point value. - fn as_f32(&self) -> f32; + fn to_f32(&self) -> f32; /// The metric closest to the given floating-point value. fn from_f32(value: f32) -> Self; }