From 119431f967fc3bd834e885cadcedb567a5392b47 Mon Sep 17 00:00:00 2001 From: mgravell Date: Wed, 16 Sep 2026 23:06:11 +0100 Subject: [PATCH] Add RedisValue.EqualityComparer, with an opt-in binary reading Comparing and hashing a RedisValue works on the text a value decodes to, which is what lets a blob equal the string it spells. For callers slinging large payloads that reading costs more than they want, and for them the bytes would do just as well. Rather than change what the type itself means - see #3229, where doing so turned out to break a Hashtable keyed by RedisValue and probed by string, silently - offer the byte reading as something a caller asks for: new Dictionary(RedisValue.EqualityComparer.Binary) An abstract base with Default and Binary, implementing both the typed and the untyped comparer interfaces; the untyped side accepts anything RedisValue itself accepts from object - string, byte[], the numerics - via the same forgiving TryParse that Equals(object) uses. Derivation is closed via a private protected constructor: nobody needs to derive from this to write their own rules, and leaving it open would fix the shape forever. Deliberately equality only. A total order would have to choose between the numeric, textual and raw-byte readings of a value, and those disagree; Redis does not define one either, since sorted sets order by score and ZRANGEBYLEX orders member bytes. Binary agrees with Default wherever a value's UTF8 form round-trips - all well-formed text - and differs where it does not, in both directions: a string holding an unpaired surrogate encodes as U+FFFD does, and a blob that is not canonical UTF8 decodes to U+FFFD without re-encoding to itself. Both its equality and its hashing read the same bytes, so it stays self-consistent; that is what makes the byte reading safe here and not on the type itself, whose GetHashCode hashes decoded text. Measured against Default on the same matrix (net10.0, string vs byte[]): 16B equal 56.3ns -> 10.8ns 1KB equal 131.9ns -> 41.1ns 1KB differs first 54.9ns -> 15.2ns 64KB equal 5,397ns -> 2,155ns 64KB differs first 52.4ns -> 15.5ns hash 64KB blob 31,026ns -> 884ns Two things the benchmarks caught that the tests could not: comparing a string against a blob has to encode a chunk at a time onto the blob's own bytes, or a mismatch in the first byte still costs a full pass over both sides; and the length precheck has to come after that, because measuring a string's UTF8 length walks all of it and dominates everything else. --- .../PublicAPI/PublicAPI.Unshipped.txt | 5 + .../RedisValue.EqualityComparer.cs | 203 ++++++++++++++++++ src/StackExchange.Redis/RedisValue.cs | 2 +- .../StackExchange.Redis.csproj | 1 + .../RedisValueEqualityBenchmarks.cs | 12 ++ .../RedisValueEquivalencyTests.cs | 134 ++++++++++++ 6 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 src/StackExchange.Redis/RedisValue.EqualityComparer.cs diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index ab058de62..986521031 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,6 @@ #nullable enable +abstract StackExchange.Redis.RedisValue.EqualityComparer.Equals(StackExchange.Redis.RedisValue x, StackExchange.Redis.RedisValue y) -> bool +abstract StackExchange.Redis.RedisValue.EqualityComparer.GetHashCode(StackExchange.Redis.RedisValue obj) -> int +StackExchange.Redis.RedisValue.EqualityComparer +static StackExchange.Redis.RedisValue.EqualityComparer.Binary.get -> StackExchange.Redis.RedisValue.EqualityComparer! +static StackExchange.Redis.RedisValue.EqualityComparer.Default.get -> StackExchange.Redis.RedisValue.EqualityComparer! diff --git a/src/StackExchange.Redis/RedisValue.EqualityComparer.cs b/src/StackExchange.Redis/RedisValue.EqualityComparer.cs new file mode 100644 index 000000000..b33463d03 --- /dev/null +++ b/src/StackExchange.Redis/RedisValue.EqualityComparer.cs @@ -0,0 +1,203 @@ +using System; +using System.Buffers; +using System.Collections; +using System.Collections.Generic; +using System.IO.Hashing; +using System.Text; + +namespace StackExchange.Redis +{ + public readonly partial struct RedisValue + { + /// + /// Ways of testing instances for equality. + /// + /// + /// + /// Equality only. There is deliberately no ordering counterpart: a total order over + /// would have to choose between the numeric, textual and raw-byte readings of + /// a value, and those disagree for the same logical value. Redis does not define one either - sorted + /// sets order by score, and ZRANGEBYLEX orders member bytes rather than values. + /// + /// + /// Derivation is closed. Anything wanting its own rules should implement + /// directly - it never needed this base class - and keeping the + /// hierarchy closed leaves room to add members here later without breaking anyone. + /// + /// + public abstract class EqualityComparer : IEqualityComparer, IEqualityComparer + { + private protected EqualityComparer() { } + + /// + /// Matches 's own equality, where a blob equals the text it decodes to. + /// + public static EqualityComparer Default { get; } = new DefaultComparer(); + + /// + /// Compares the UTF-8 form of values without decoding it into text, which is markedly cheaper for + /// large values. + /// + /// + /// + /// Agrees with wherever a value's UTF-8 form round-trips, which is to say + /// for all well-formed text. It differs where that fails, and either side can be the cause: + /// + /// + /// + /// A string holding an unpaired surrogate encodes to the same bytes as one holding U+FFFD, so + /// this calls those equal where does not. + /// + /// + /// A blob that is not canonical UTF-8 decodes to U+FFFD but does not re-encode to itself - the + /// single byte 0xFF, say - so this calls it distinct from the text U+FFFD where + /// calls them equal. + /// + /// + /// + /// Since both its equality and its hashing read those same bytes, it is self-consistent - which + /// is what makes the byte reading safe here and not on itself, whose + /// hashes the decoded text. + /// + /// + /// Hashing is not resistant to deliberate collision-finding - it is chosen for speed, unlike the + /// framework's string hashing. Do not use it to key on values an untrusted party controls. + /// + /// + public static EqualityComparer Binary { get; } = new BinaryComparer(); + + /// + public abstract bool Equals(RedisValue x, RedisValue y); + + /// + public abstract int GetHashCode(RedisValue obj); + + // The untyped API accepts anything RedisValue itself would accept from object - string, byte[], + // the numeric types and so on - matching Equals(object) rather than demanding a boxed RedisValue. + bool IEqualityComparer.Equals(object? x, object? y) + { + if (ReferenceEquals(x, y)) return true; // also covers both-null + + var left = TryParse(x, out var leftValid); + var right = TryParse(y, out var rightValid); + return leftValid && rightValid && Equals(left, right); + } + + int IEqualityComparer.GetHashCode(object obj) + { + var value = TryParse(obj, out var valid); + + // anything we cannot read is never equal to anything under Equals above, so its own hash is + // as good as any: it only has to be stable + return valid ? GetHashCode(value) : obj.GetHashCode(); + } + + private sealed class DefaultComparer : EqualityComparer + { + public override bool Equals(RedisValue x, RedisValue y) => x == y; + + public override int GetHashCode(RedisValue obj) => obj.GetHashCode(); + } + + private sealed class BinaryComparer : EqualityComparer + { + /// Per-process entropy, so hash codes are not predictable between runs. + private static readonly long Seed = BitConverter.ToInt64(Guid.NewGuid().ToByteArray(), 0); + + private const int StackLimit = 256; + + public override bool Equals(RedisValue x, RedisValue y) + { + if (x.IsNull || y.IsNull) return x.IsNull && y.IsNull; + + // byte-backed on both sides: the bytes are already there, so no copy is needed + if (IsBlob(x.Type) && IsBlob(y.Type)) return BlobSequenceEqual(x, y); + + // A string against a contiguous blob is the case worth caring about: encode the string a + // chunk at a time straight onto the blob's own bytes, so a mismatch near the front stops + // there instead of after both sides have been written out in full. Deliberately ahead of + // any length check - measuring the string's UTF8 length means walking all of it, which + // costs more than the comparison usually does. + if (x.Type == StorageType.String && IsContiguousBlob(y.Type)) return StringEqualsBytes(x.RawString(), y.UnsafeRawSpan(out _)); + if (y.Type == StorageType.String && IsContiguousBlob(x.Type)) return StringEqualsBytes(y.RawString(), x.UnsafeRawSpan(out _)); + + int length = x.GetByteCount(); + if (length != y.GetByteCount()) return false; + if (length == 0) return true; + + byte[]? leasedX = null, leasedY = null; + Span bytesX = length <= StackLimit ? stackalloc byte[StackLimit] : (leasedX = ArrayPool.Shared.Rent(length)); + Span bytesY = length <= StackLimit ? stackalloc byte[StackLimit] : (leasedY = ArrayPool.Shared.Rent(length)); + + x.CopyTo(bytesX); + y.CopyTo(bytesY); + bool equal = bytesX.Slice(0, length).SequenceEqual(bytesY.Slice(0, length)); + + if (leasedX is not null) ArrayPool.Shared.Return(leasedX); + if (leasedY is not null) ArrayPool.Shared.Return(leasedY); + return equal; + } + + private static bool IsContiguousBlob(StorageType type) + => type is StorageType.ByteArray or StorageType.MemoryManager or StorageType.ShortBlob; + + /// + /// Compares a string's UTF-8 form against bytes, encoding it a chunk at a time so that a + /// mismatch costs only the chunk that contains it. + /// + private static bool StringEqualsBytes(string s, scoped ReadOnlySpan utf8) + { + const int ChunkChars = 512; + Span buffer = stackalloc byte[ChunkChars * MaxUtf8BytesPerChar]; + + var chars = s.AsSpan(); + while (!chars.IsEmpty) + { + var take = Math.Min(ChunkChars, chars.Length); + + // never split a surrogate pair: the encoder would emit U+FFFD for each half, which is + // not what encoding the whole string would have produced + if (take < chars.Length && char.IsHighSurrogate(chars[take - 1])) take++; + + var written = Encoding.UTF8.GetBytes(chars.Slice(0, take), buffer); + if (written > utf8.Length || !buffer.Slice(0, written).SequenceEqual(utf8.Slice(0, written))) return false; + + chars = chars.Slice(take); + utf8 = utf8.Slice(written); + } + return utf8.IsEmpty; + } + + /// Worst case UTF-8 bytes for a single char (a lone surrogate becomes U+FFFD). + private const int MaxUtf8BytesPerChar = 3; + + public override int GetHashCode(RedisValue obj) + { + if (obj.IsNull) return -1; + + switch (obj.Type) + { + case StorageType.ByteArray or StorageType.MemoryManager or StorageType.ShortBlob: + return Fold(XxHash3.HashToUInt64(obj.UnsafeRawSpan(out _), Seed)); + case StorageType.Sequence: + var sequence = obj.RawSequence(); + if (sequence.IsSingleSegment) return Fold(XxHash3.HashToUInt64(sequence.First.Span, Seed)); + break; // multi-segment: fall through to the copy below rather than streaming + } + + int length = obj.GetByteCount(); + if (length == 0) return 0; + + byte[]? leased = null; + Span bytes = length <= StackLimit ? stackalloc byte[StackLimit] : (leased = ArrayPool.Shared.Rent(length)); + obj.CopyTo(bytes); + var hash = Fold(XxHash3.HashToUInt64(bytes.Slice(0, length), Seed)); + if (leased is not null) ArrayPool.Shared.Return(leased); + return hash; + } + + private static int Fold(ulong hash) => unchecked((int)hash ^ (int)(hash >> 32)); + } + } + } +} diff --git a/src/StackExchange.Redis/RedisValue.cs b/src/StackExchange.Redis/RedisValue.cs index 3c2c2c22d..30634041f 100644 --- a/src/StackExchange.Redis/RedisValue.cs +++ b/src/StackExchange.Redis/RedisValue.cs @@ -22,7 +22,7 @@ namespace StackExchange.Redis /// Represents values that can be stored in redis. /// [StructLayout(LayoutKind.Explicit)] - public readonly struct RedisValue : IEquatable, IComparable, IComparable, IConvertible + public readonly partial struct RedisValue : IEquatable, IComparable, IComparable, IConvertible { // Maximum payload that fits in an inline short-blob (packed into the overlapped int64 field). internal const int MaxInlineBytes = sizeof(long); diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 9bb67d90a..2b20036a9 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -42,6 +42,7 @@ + diff --git a/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs b/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs index de695b48f..e7aa4cc6c 100644 --- a/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs +++ b/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs @@ -108,6 +108,18 @@ public void Setup() [Benchmark] public int HashString() => _string.GetHashCode(); + /// The opt-in byte comparer on the same mixed case: no decode at all. + [Benchmark] + public bool BinaryStringVsByteArray() => RedisValue.EqualityComparer.Binary.Equals(_string, _byteArray); + + /// The opt-in byte comparer, blob against blob. + [Benchmark] + public bool BinaryBlobVsBlob() => RedisValue.EqualityComparer.Binary.Equals(_other, _byteArray); + + /// Hashing a blob through the byte comparer: raw bytes, never decoded. + [Benchmark] + public int BinaryHashBlob() => RedisValue.EqualityComparer.Binary.GetHashCode(_byteArray); + private sealed class Segment : ReadOnlySequenceSegment { public Segment(ReadOnlyMemory value, Segment? head) diff --git a/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs b/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs index de97ca184..373a9cefa 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs @@ -1,5 +1,7 @@ using System; using System.Buffers; +using System.Collections; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Text; using Xunit; @@ -766,4 +768,136 @@ public EquivalenceSegment(ReadOnlyMemory value, EquivalenceSegment? head) } } } + + [Fact] + public void DefaultComparer_MatchesTheTypesOwnEquality() + { + var comparer = RedisValue.EqualityComparer.Default; + foreach (var blob in EquivalenceBlobs()) + { + RedisValue asBlob = blob; + foreach (var s in EquivalenceStrings()) + { + RedisValue asString = s; + var because = $"'{Escape(s)}' vs {BitConverter.ToString(blob)}"; + + Assert.True((asString == asBlob) == comparer.Equals(asString, asBlob), because); + Assert.Equal(asString.GetHashCode(), comparer.GetHashCode(asString)); + Assert.Equal(asBlob.GetHashCode(), comparer.GetHashCode(asBlob)); + } + } + } + + [Fact] + public void BinaryComparer_AgreesWithDefaultOnWellFormedText() + { + var binary = RedisValue.EqualityComparer.Binary; + foreach (var blob in EquivalenceBlobs()) + { + RedisValue asBlob = blob; + foreach (var s in EquivalenceStrings()) + { + // The two readings may differ exactly where UTF8 does not round-trip, and either side can be + // the culprit: a string holding an unpaired surrogate, or a blob that is not canonical UTF8 + // (0xFF decodes to U+FFFD but re-encodes to EF BF BD). Everywhere else they must agree. + if (Encoding.UTF8.GetString(Encoding.UTF8.GetBytes(s)) != s) continue; + if (!Encoding.UTF8.GetBytes(Encoding.UTF8.GetString(blob)).AsSpan().SequenceEqual(blob)) continue; + + RedisValue asString = s; + Assert.True( + (asString == asBlob) == binary.Equals(asString, asBlob), + $"'{Escape(s)}' vs {BitConverter.ToString(blob)}"); + } + } + } + + [Fact] + public void BinaryComparer_DivergesOnlyWhereUtf8DoesNotRoundTrip() + { + var binary = RedisValue.EqualityComparer.Binary; + + // a lone surrogate encodes to the same bytes as U+FFFD, so Binary calls them equal and the default + // does not - this is the documented divergence, pinned so it cannot drift silently + RedisValue loneSurrogate = new string([(char)0xD800]); + RedisValue replacement = "�"; + + Assert.False(loneSurrogate == replacement); + Assert.True(binary.Equals(loneSurrogate, replacement)); + + // and Binary stays self-consistent about it: equal means same hash + Assert.Equal(binary.GetHashCode(loneSurrogate), binary.GetHashCode(replacement)); + + // the other direction, where the *blob* is not canonical UTF8: 0xFF decodes to U+FFFD, so the default + // calls it equal to that text, while Binary sees FF against EF BF BD and does not + RedisValue invalidBlob = new byte[] { 0xFF }; + Assert.True(replacement == invalidBlob); + Assert.False(binary.Equals(replacement, invalidBlob)); + } + + [Fact] + public void BinaryComparer_EqualValuesShareHashCodes() + { + var binary = RedisValue.EqualityComparer.Binary; + foreach (var blob in EquivalenceBlobs()) + { + RedisValue asBlob = blob; + RedisValue asSegmented = blob.Length >= 2 ? Segmented(blob, blob.Length / 2) : asBlob; + + foreach (var s in EquivalenceStrings()) + { + RedisValue asString = s; + if (binary.Equals(asString, asBlob)) + { + Assert.True( + binary.GetHashCode(asString) == binary.GetHashCode(asBlob), + $"equal under Binary must share a hash: '{Escape(s)}' vs {BitConverter.ToString(blob)}"); + } + } + + // representation must not matter: the same bytes, contiguous or segmented + Assert.True(binary.Equals(asBlob, asSegmented)); + Assert.Equal(binary.GetHashCode(asBlob), binary.GetHashCode(asSegmented)); + } + } + + [Theory] + [InlineData("abc")] + [InlineData("42")] + public void Comparers_UntypedApiAcceptsWhateverRedisValueAccepts(string value) + { + foreach (var comparer in new[] { RedisValue.EqualityComparer.Default, RedisValue.EqualityComparer.Binary }) + { + var untyped = (IEqualityComparer)comparer; + object asString = value; + object asBytes = Encoding.UTF8.GetBytes(value); + object asRedisValue = (RedisValue)value; + + Assert.True(untyped.Equals(asString, asBytes)); + Assert.True(untyped.Equals(asString, asRedisValue)); + Assert.True(untyped.Equals(asBytes, asRedisValue)); + + Assert.Equal(untyped.GetHashCode(asString), untyped.GetHashCode(asBytes)); + Assert.Equal(untyped.GetHashCode(asString), untyped.GetHashCode(asRedisValue)); + + // something it cannot read is not equal to anything, but is still reflexive and stable + object unreadable = new object(); + Assert.False(untyped.Equals(unreadable, asString)); + Assert.True(untyped.Equals(unreadable, unreadable)); + Assert.Equal(unreadable.GetHashCode(), untyped.GetHashCode(unreadable)); + } + } + + [Fact] + public void BinaryComparer_WorksAsADictionaryComparer() + { + var dictionary = new Dictionary(RedisValue.EqualityComparer.Binary) + { + { "alpha", "one" }, + { Encoding.UTF8.GetBytes("beta"), "two" }, + }; + + Assert.Equal("one", dictionary[Encoding.UTF8.GetBytes("alpha")]); + Assert.Equal("two", dictionary["beta"]); + Assert.False(dictionary.ContainsKey("gamma")); + } }