From 7272c3447db58bcc57353c097d424ba92308e3f1 Mon Sep 17 00:00:00 2001 From: fleisch Date: Fri, 21 Aug 2026 11:30:50 +0200 Subject: [PATCH 1/2] Normalize index keys to double only where the conversion is exact DBValue folds every non-Double number to a double so that Integer(5) and Double(5.0) share an index key, which stores comparing the encoded key bytes need in order to match across types (gh-178). A double only steps by one up to 2^53 though. Around 8.7e17, where snowflake ids and TSIDs live, the representable doubles are 128 apart, so ids closer than that become one key: a unique index rejects an id that is not a duplicate, and a non-unique lookup returns rows belonging to a neighbour. Keep the fold, but only where the value survives it. Integer, Short, Byte and Float always do; Long, BigInteger and BigDecimal are compared against the exact value of the double they produce and keep their own type when it differs. Cross-type equality is unchanged over the range gh-178 is about. --- .../java/org/dizitart/no2/common/DBValue.java | 40 ++++++++- .../org/dizitart/no2/common/DBValueTest.java | 83 +++++++++++++++++++ .../CollectionLargeIdIndexTest.java | 62 ++++++++++++++ 3 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 nitrite/src/test/java/org/dizitart/no2/common/DBValueTest.java create mode 100644 nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java diff --git a/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java b/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java index 2294c7373..c6545d3e5 100644 --- a/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java +++ b/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java @@ -26,6 +26,8 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; /** * @author Anindya Chatterjee @@ -59,14 +61,48 @@ public int compareTo(DBValue o) { } private static Comparable normalizeNumber(Comparable value) { - // Normalize all numeric types to Double for consistent serialization + // Normalize numeric types to Double for consistent serialization // This ensures Integer(5) and Double(5.0) are treated the same in indexes if (value instanceof Number && !(value instanceof Double)) { - return ((Number) value).doubleValue(); + double normalized = ((Number) value).doubleValue(); + // ...but only where a double can hold the value exactly. Beyond 2^53 it cannot, + // and folding there maps distinct numbers onto one index key: consecutive longs + // around 8.7e17 are 128 apart as doubles, so ids closer than that become the same + // key, which makes a unique index reject a new id and a non-unique one return rows + // belonging to a different id. + if (isExactAsDouble((Number) value, normalized)) { + return normalized; + } } return value; } + private static boolean isExactAsDouble(Number value, double normalized) { + if (value instanceof Integer || value instanceof Short + || value instanceof Byte || value instanceof Float) { + // every value of these types survives the widening unchanged + return true; + } + + if (Double.isNaN(normalized) || Double.isInfinite(normalized)) { + return false; + } + + // new BigDecimal(double) is the exact value of the double, so this compares the + // number against what the conversion actually produced + BigDecimal converted = new BigDecimal(normalized); + if (value instanceof Long) { + return converted.compareTo(BigDecimal.valueOf(value.longValue())) == 0; + } + if (value instanceof BigInteger) { + return converted.compareTo(new BigDecimal((BigInteger) value)) == 0; + } + if (value instanceof BigDecimal) { + return converted.compareTo((BigDecimal) value) == 0; + } + return false; + } + private void writeObject(ObjectOutputStream stream) throws IOException { stream.writeObject(value); } diff --git a/nitrite/src/test/java/org/dizitart/no2/common/DBValueTest.java b/nitrite/src/test/java/org/dizitart/no2/common/DBValueTest.java new file mode 100644 index 000000000..aa4876274 --- /dev/null +++ b/nitrite/src/test/java/org/dizitart/no2/common/DBValueTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.common; + +import org.junit.Test; + +import java.math.BigInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +public class DBValueTest { + + @Test + public void testSmallNumbersAreNormalizedToDouble() { + // cross-type equality for values a double holds exactly, including in stores that + // compare the encoded key rather than going through compareTo + assertEquals(new DBValue(5.0), new DBValue(5)); + assertEquals(new DBValue(5.0), new DBValue(5L)); + assertEquals(new DBValue(5.0), new DBValue((short) 5)); + assertEquals(new DBValue(5.0), new DBValue((byte) 5)); + assertEquals(new DBValue(5.0), new DBValue(BigInteger.valueOf(5))); + } + + @Test + public void testLargeLongsKeepTheirValue() { + long id = 870000000000000123L; // beyond 2^53, doubles are 128 apart here + assertEquals(id, new DBValue(id).getValue()); + } + + @Test + public void testLongsCloserThanDoublePrecisionStayDistinct() { + long id = 870000000000000123L; + assertNotEquals(new DBValue(id), new DBValue(id + 1)); + assertNotEquals(0, new DBValue(id).compareTo(new DBValue(id + 1))); + } + + @Test + public void testLargeBigIntegerKeepsItsValue() { + // odd and far beyond 2^53, so no double holds it exactly + BigInteger value = BigInteger.ONE.shiftLeft(70).add(BigInteger.ONE); + assertEquals(value, new DBValue(value).getValue()); + assertNotEquals(new DBValue(value), new DBValue(value.add(BigInteger.valueOf(2)))); + } + + @Test + public void testLongAtTheEdgeOfExactRange() { + long exact = 1L << 53; // the largest power of two a double still steps by one + assertEquals(2.0 * (1L << 52), new DBValue(exact).getValue()); + // one above it is not representable, so it has to keep its own value + assertEquals(exact + 1, new DBValue(exact + 1).getValue()); + } + + @Test + public void testExactlyRepresentableLargeValuesStillNormalize() { + // 2^63 is a power of two, so the conversion loses nothing and folding is safe + BigInteger powerOfTwo = BigInteger.ONE.shiftLeft(63); + assertEquals(Math.pow(2, 63), new DBValue(powerOfTwo).getValue()); + } + + @Test + public void testNumbersStillCompareAcrossTypes() { + // compareTo goes through Comparables/Numbers, so this holds whatever the stored form is + long id = 870000000000000123L; + assertEquals(0, new DBValue(id).compareTo(new DBValue(BigInteger.valueOf(id)))); + assertEquals(0, new DBValue(5).compareTo(new DBValue(5.0))); + } +} diff --git a/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java new file mode 100644 index 000000000..843203fc4 --- /dev/null +++ b/nitrite/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.collection; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.index.IndexType; +import org.junit.Test; + +import static org.dizitart.no2.collection.Document.createDocument; +import static org.dizitart.no2.filters.FluentFilter.where; +import static org.dizitart.no2.index.IndexOptions.indexOptions; +import static org.junit.Assert.assertEquals; + +/** + * Ids above 2^53 - snowflake ids, TSIDs and the like - are further apart than a double can + * step, so an index that keyed them as doubles could not tell them apart. + */ +public class CollectionLargeIdIndexTest extends BaseCollectionTest { + + // two ids 1 apart; the nearest doubles around here are 128 apart + private static final long FIRST_ID = 870000000000000123L; + private static final long SECOND_ID = FIRST_ID + 1; + + @Test + public void testUniqueIndexAcceptsIdsCloserThanDoublePrecision() { + collection.remove(org.dizitart.no2.filters.Filter.ALL); + collection.createIndex(indexOptions(IndexType.UNIQUE), "entityId"); + + collection.insert(createDocument("entityId", FIRST_ID)); + collection.insert(createDocument("entityId", SECOND_ID)); + + assertEquals(2, collection.find().size()); + } + + @Test + public void testIndexedLookupReturnsOnlyTheMatchingId() { + collection.remove(org.dizitart.no2.filters.Filter.ALL); + collection.createIndex(indexOptions(IndexType.NON_UNIQUE), "entityId"); + + collection.insert(createDocument("entityId", FIRST_ID)); + collection.insert(createDocument("entityId", SECOND_ID)); + + Document found = collection.find(where("entityId").eq(FIRST_ID)).firstOrNull(); + assertEquals(1, collection.find(where("entityId").eq(FIRST_ID)).size()); + assertEquals(FIRST_ID, (long) found.get("entityId", Long.class)); + } +} From 8bdad5b23a998a4dbc9459a8847a95417cdd2337 Mon Sep 17 00:00:00 2001 From: Anindya Chatterjee Date: Mon, 31 Aug 2026 14:39:11 +0530 Subject: [PATCH 2/2] Take the exactness check off the allocation path, and cover it on RocksDB isExactAsDouble built a BigDecimal for every Long index key to decide whether the fold was lossless - including the small ones that obviously survive it. Casting the double back is exact for every double inside long range, so the round trip answers the same question without allocating; the range check is what keeps Long.MAX_VALUE honest, since its double rounds up to 2^63 and the cast back saturates onto MAX_VALUE again. Verified against the BigDecimal version over 25M values including every power-of-two boundary: no divergence. Measured 30.9ns -> 4.7ns per key, though end to end it is within the noise of an insert - this is about not allocating per index key, not about the clock. CollectionLargeIdIndexTest also runs on RocksDB now. The fold being narrowed exists for byte-comparing stores in the first place, so that is the store where a change to the stored form actually shows: without the fix both cases fail there exactly as they do in memory. Co-Authored-By: Claude Opus 5 --- .../CollectionLargeIdIndexTest.java | 66 +++++++++++++++++++ .../java/org/dizitart/no2/common/DBValue.java | 12 +++- 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java diff --git a/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java new file mode 100644 index 000000000..2ae3f1351 --- /dev/null +++ b/nitrite-rocksdb-adapter/src/test/java/org/dizitart/no2/integration/collection/CollectionLargeIdIndexTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017-2021 Nitrite author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dizitart.no2.integration.collection; + +import org.dizitart.no2.collection.Document; +import org.dizitart.no2.index.IndexType; +import org.junit.Test; + +import static org.dizitart.no2.collection.Document.createDocument; +import static org.dizitart.no2.filters.FluentFilter.where; +import static org.dizitart.no2.index.IndexOptions.indexOptions; +import static org.junit.Assert.assertEquals; + +/** + * Ids above 2^53 - snowflake ids, TSIDs and the like - are further apart than a double can + * step, so an index that keyed them as doubles could not tell them apart. + * + *

Run against RocksDB as well as the in-memory store, because the fold this narrows exists + * for byte-comparing stores in the first place: RocksDB matches on the encoded key rather than + * through {@code compareTo}, so it is where a change to the stored form actually shows. + */ +public class CollectionLargeIdIndexTest extends BaseCollectionTest { + + // two ids 1 apart; the nearest doubles around here are 128 apart + private static final long FIRST_ID = 870000000000000123L; + private static final long SECOND_ID = FIRST_ID + 1; + + @Test + public void testUniqueIndexAcceptsIdsCloserThanDoublePrecision() { + collection.remove(org.dizitart.no2.filters.Filter.ALL); + collection.createIndex(indexOptions(IndexType.UNIQUE), "entityId"); + + collection.insert(createDocument("entityId", FIRST_ID)); + collection.insert(createDocument("entityId", SECOND_ID)); + + assertEquals(2, collection.find().size()); + } + + @Test + public void testIndexedLookupReturnsOnlyTheMatchingId() { + collection.remove(org.dizitart.no2.filters.Filter.ALL); + collection.createIndex(indexOptions(IndexType.NON_UNIQUE), "entityId"); + + collection.insert(createDocument("entityId", FIRST_ID)); + collection.insert(createDocument("entityId", SECOND_ID)); + + Document found = collection.find(where("entityId").eq(FIRST_ID)).firstOrNull(); + assertEquals(1, collection.find(where("entityId").eq(FIRST_ID)).size()); + assertEquals(FIRST_ID, (long) found.get("entityId", Long.class)); + } +} diff --git a/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java b/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java index c6545d3e5..211c05ba3 100644 --- a/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java +++ b/nitrite/src/main/java/org/dizitart/no2/common/DBValue.java @@ -84,6 +84,15 @@ private static boolean isExactAsDouble(Number value, double normalized) { return true; } + if (value instanceof Long) { + // Casting the double back is exact for every double inside long range, so a value + // that survives the round trip is one the double holds exactly. The range check is + // what keeps Long.MAX_VALUE honest: its double rounds up to 2^63, and the cast back + // saturates onto MAX_VALUE again, which would otherwise read as exact. + long exact = value.longValue(); + return normalized >= -0x1p63 && normalized < 0x1p63 && (long) normalized == exact; + } + if (Double.isNaN(normalized) || Double.isInfinite(normalized)) { return false; } @@ -91,9 +100,6 @@ private static boolean isExactAsDouble(Number value, double normalized) { // new BigDecimal(double) is the exact value of the double, so this compares the // number against what the conversion actually produced BigDecimal converted = new BigDecimal(normalized); - if (value instanceof Long) { - return converted.compareTo(BigDecimal.valueOf(value.longValue())) == 0; - } if (value instanceof BigInteger) { return converted.compareTo(new BigDecimal((BigInteger) value)) == 0; }