diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index ab058de62..986521031 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,6 @@ #nullable enable +abstract StackExchange.Redis.RedisValue.EqualityComparer.Equals(StackExchange.Redis.RedisValue x, StackExchange.Redis.RedisValue y) -> bool +abstract StackExchange.Redis.RedisValue.EqualityComparer.GetHashCode(StackExchange.Redis.RedisValue obj) -> int +StackExchange.Redis.RedisValue.EqualityComparer +static StackExchange.Redis.RedisValue.EqualityComparer.Binary.get -> StackExchange.Redis.RedisValue.EqualityComparer! +static StackExchange.Redis.RedisValue.EqualityComparer.Default.get -> StackExchange.Redis.RedisValue.EqualityComparer! diff --git a/src/StackExchange.Redis/RedisValue.EqualityComparer.cs b/src/StackExchange.Redis/RedisValue.EqualityComparer.cs new file mode 100644 index 000000000..9a6d4d82d --- /dev/null +++ b/src/StackExchange.Redis/RedisValue.EqualityComparer.cs @@ -0,0 +1,243 @@ +using System; +using System.Buffers; +using System.Collections; +using System.Collections.Generic; +using System.IO.Hashing; +using System.Security.Cryptography; +using System.Text; + +namespace StackExchange.Redis +{ + public readonly partial struct RedisValue + { + /// + /// Ways of testing instances for equality. + /// + /// + /// + /// Equality only. There is deliberately no ordering counterpart: a total order over + /// would have to choose between the numeric, textual and raw-byte readings of + /// a value, and those disagree for the same logical value. Redis does not define one either - sorted + /// sets order by score, and ZRANGEBYLEX orders member bytes rather than values. + /// + /// + /// Derivation is closed. Anything wanting its own rules should implement + /// directly - it never needed this base class - and keeping the + /// hierarchy closed leaves room to add members here later without breaking anyone. + /// + /// + public abstract class EqualityComparer : IEqualityComparer, IEqualityComparer + { + private protected EqualityComparer() { } + + /// + /// Matches 's own equality, where a blob equals the text it decodes to. + /// + public static EqualityComparer Default { get; } = new DefaultComparer(); + + /// + /// Compares the UTF-8 form of values without decoding it into text, which is markedly cheaper for + /// large values. + /// + /// + /// + /// Agrees with for text that is well-formed and not numeric. It differs in + /// three places: + /// + /// + /// + /// Numeric text. reduces anything that parses as a number before + /// comparing, so "1.0", "1.00" and "1" are all equal to it, as are "0" + /// and "-0.0". This compares the bytes, so they are not - which is how the server + /// identifies members, and usually what a byte reading is wanted for. + /// + /// + /// Unpaired surrogates. A string holding one encodes to the same bytes as one holding + /// U+FFFD, so this calls those equal where does not. + /// + /// + /// Non-canonical UTF-8. A blob that does not re-encode to itself - the single byte + /// 0xFF, say, which decodes to U+FFFD - is distinct from that text here, where + /// calls them equal. + /// + /// + /// + /// Since both its equality and its hashing read those same bytes, it is self-consistent - which + /// is what makes the byte reading safe here and not on itself, whose + /// hashes the decoded text. + /// + /// + /// It also does not care how a value is stored: the same text compares equal to itself whether it + /// arrived as a string or as the bytes of that string, which is not true of the default rules for + /// every input. + /// + /// + /// Hashing is not resistant to deliberate collision-finding - it is chosen for speed, unlike the + /// framework's string hashing. Do not use it to key on values an untrusted party controls. + /// + /// + public static EqualityComparer Binary { get; } = new BinaryComparer(); + + /// + public abstract bool Equals(RedisValue x, RedisValue y); + + /// + public abstract int GetHashCode(RedisValue obj); + + // The untyped API accepts anything RedisValue itself would accept from object - string, byte[], + // the numeric types and so on - matching Equals(object) rather than demanding a boxed RedisValue. + bool IEqualityComparer.Equals(object? x, object? y) + { + if (ReferenceEquals(x, y)) return true; // also covers both-null + + var left = TryParse(x, out var leftValid); + var right = TryParse(y, out var rightValid); + return leftValid && rightValid && Equals(left, right); + } + + int IEqualityComparer.GetHashCode(object obj) + { + var value = TryParse(obj, out var valid); + + // anything we cannot read is never equal to anything under Equals above, so its own hash is + // as good as any: it only has to be stable + return valid ? GetHashCode(value) : obj.GetHashCode(); + } + + private sealed class DefaultComparer : EqualityComparer + { + public override bool Equals(RedisValue x, RedisValue y) => x == y; + + public override int GetHashCode(RedisValue obj) => obj.GetHashCode(); + } + + private sealed class BinaryComparer : EqualityComparer + { + /// + /// Per-process entropy, so hash codes are not predictable between runs. Not from + /// : its version and variant bits are fixed, so the first eight + /// bytes carry slightly less than they appear to. + /// + private static readonly long Seed = ReadSeed(); + + private static long ReadSeed() + { + // the array form rather than Fill(Span), which the down-level targets lack; this + // runs once per process + var bytes = new byte[sizeof(long)]; + using var rng = RandomNumberGenerator.Create(); + rng.GetBytes(bytes); + return BitConverter.ToInt64(bytes, 0); + } + + private const int StackLimit = 256; + + public override bool Equals(RedisValue x, RedisValue y) + { + if (x.IsNull || y.IsNull) return x.IsNull && y.IsNull; + + // byte-backed on both sides: the bytes are already there, so no copy is needed + if (IsBlob(x.Type) && IsBlob(y.Type)) return BlobSequenceEqual(x, y); + + // identical text encodes identically, so this is a shortcut rather than a rule; the + // unequal case still has to go the byte route, because two different strings can share a + // UTF-8 form once unpaired surrogates are involved + if (x.Type == StorageType.String && y.Type == StorageType.String + && string.Equals(x.RawString(), y.RawString(), StringComparison.Ordinal)) + { + return true; + } + + // A string against a contiguous blob is the case worth caring about: encode the string a + // chunk at a time straight onto the blob's own bytes, so a mismatch near the front stops + // there instead of after both sides have been written out in full. Deliberately ahead of + // any length check - measuring the string's UTF8 length means walking all of it, which + // costs more than the comparison usually does. + if (x.Type == StorageType.String && IsContiguousBlob(y.Type)) return StringEqualsBytes(x.RawString(), y.UnsafeRawSpan(out _)); + if (y.Type == StorageType.String && IsContiguousBlob(x.Type)) return StringEqualsBytes(y.RawString(), x.UnsafeRawSpan(out _)); + + int length = x.GetByteCount(); + if (length != y.GetByteCount()) return false; + if (length == 0) return true; + + byte[]? leasedX = null, leasedY = null; + Span bytesX = length <= StackLimit ? stackalloc byte[StackLimit] : (leasedX = ArrayPool.Shared.Rent(length)); + Span bytesY = length <= StackLimit ? stackalloc byte[StackLimit] : (leasedY = ArrayPool.Shared.Rent(length)); + + x.CopyTo(bytesX); + y.CopyTo(bytesY); + bool equal = bytesX.Slice(0, length).SequenceEqual(bytesY.Slice(0, length)); + + if (leasedX is not null) ArrayPool.Shared.Return(leasedX); + if (leasedY is not null) ArrayPool.Shared.Return(leasedY); + return equal; + } + + private static bool IsContiguousBlob(StorageType type) + => type is StorageType.ByteArray or StorageType.MemoryManager or StorageType.ShortBlob; + + /// + /// Compares a string's UTF-8 form against bytes, encoding it a chunk at a time so that a + /// mismatch costs only the chunk that contains it. + /// + private static bool StringEqualsBytes(string s, scoped ReadOnlySpan utf8) + { + const int ChunkChars = 512; + Span buffer = stackalloc byte[ChunkChars * MaxUtf8BytesPerChar]; + + var chars = s.AsSpan(); + while (!chars.IsEmpty) + { + var take = Math.Min(ChunkChars, chars.Length); + + // Never end a chunk on a high surrogate: the encoder would emit U+FFFD for each half + // of a pair split across chunks, which is not what encoding the whole string gives. + // Step *back* rather than forward - taking one more character would both overrun the + // buffer (513 three-byte characters do not fit in 512*3 bytes) and, where the + // character at the boundary is a lone high surrogate, simply move the split onto the + // following pair instead of avoiding it. Only reachable when take == ChunkChars, so + // it cannot reach zero. + if (take < chars.Length && char.IsHighSurrogate(chars[take - 1])) take--; + + var written = Encoding.UTF8.GetBytes(chars.Slice(0, take), buffer); + if (written > utf8.Length || !buffer.Slice(0, written).SequenceEqual(utf8.Slice(0, written))) return false; + + chars = chars.Slice(take); + utf8 = utf8.Slice(written); + } + return utf8.IsEmpty; + } + + /// Worst case UTF-8 bytes for a single char (a lone surrogate becomes U+FFFD). + private const int MaxUtf8BytesPerChar = 3; + + public override int GetHashCode(RedisValue obj) + { + if (obj.IsNull) return -1; + + switch (obj.Type) + { + case StorageType.ByteArray or StorageType.MemoryManager or StorageType.ShortBlob: + return Fold(XxHash3.HashToUInt64(obj.UnsafeRawSpan(out _), Seed)); + case StorageType.Sequence: + var sequence = obj.RawSequence(); + if (sequence.IsSingleSegment) return Fold(XxHash3.HashToUInt64(sequence.First.Span, Seed)); + break; // multi-segment: fall through to the copy below rather than streaming + } + + int length = obj.GetByteCount(); + if (length == 0) return 0; + + byte[]? leased = null; + Span bytes = length <= StackLimit ? stackalloc byte[StackLimit] : (leased = ArrayPool.Shared.Rent(length)); + obj.CopyTo(bytes); + var hash = Fold(XxHash3.HashToUInt64(bytes.Slice(0, length), Seed)); + if (leased is not null) ArrayPool.Shared.Return(leased); + return hash; + } + + private static int Fold(ulong hash) => unchecked((int)hash ^ (int)(hash >> 32)); + } + } + } +} diff --git a/src/StackExchange.Redis/RedisValue.cs b/src/StackExchange.Redis/RedisValue.cs index 3c2c2c22d..30634041f 100644 --- a/src/StackExchange.Redis/RedisValue.cs +++ b/src/StackExchange.Redis/RedisValue.cs @@ -22,7 +22,7 @@ namespace StackExchange.Redis /// Represents values that can be stored in redis. /// [StructLayout(LayoutKind.Explicit)] - public readonly struct RedisValue : IEquatable, IComparable, IComparable, IConvertible + public readonly partial struct RedisValue : IEquatable, IComparable, IComparable, IConvertible { // Maximum payload that fits in an inline short-blob (packed into the overlapped int64 field). internal const int MaxInlineBytes = sizeof(long); diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 9bb67d90a..2b20036a9 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -42,6 +42,7 @@ + diff --git a/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs b/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs index de695b48f..e7aa4cc6c 100644 --- a/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs +++ b/tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs @@ -108,6 +108,18 @@ public void Setup() [Benchmark] public int HashString() => _string.GetHashCode(); + /// The opt-in byte comparer on the same mixed case: no decode at all. + [Benchmark] + public bool BinaryStringVsByteArray() => RedisValue.EqualityComparer.Binary.Equals(_string, _byteArray); + + /// The opt-in byte comparer, blob against blob. + [Benchmark] + public bool BinaryBlobVsBlob() => RedisValue.EqualityComparer.Binary.Equals(_other, _byteArray); + + /// Hashing a blob through the byte comparer: raw bytes, never decoded. + [Benchmark] + public int BinaryHashBlob() => RedisValue.EqualityComparer.Binary.GetHashCode(_byteArray); + private sealed class Segment : ReadOnlySequenceSegment { public Segment(ReadOnlyMemory value, Segment? head) diff --git a/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs b/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs index de97ca184..6f8208ea5 100644 --- a/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs +++ b/tests/StackExchange.Redis.Tests/RedisValueEquivalencyTests.cs @@ -1,5 +1,7 @@ using System; using System.Buffers; +using System.Collections; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Text; using Xunit; @@ -766,4 +768,206 @@ public EquivalenceSegment(ReadOnlyMemory value, EquivalenceSegment? head) } } } + + [Fact] + public void DefaultComparer_MatchesTheTypesOwnEquality() + { + var comparer = RedisValue.EqualityComparer.Default; + foreach (var blob in EquivalenceBlobs()) + { + RedisValue asBlob = blob; + foreach (var s in EquivalenceStrings()) + { + RedisValue asString = s; + var because = $"'{Escape(s)}' vs {BitConverter.ToString(blob)}"; + + Assert.True((asString == asBlob) == comparer.Equals(asString, asBlob), because); + Assert.Equal(asString.GetHashCode(), comparer.GetHashCode(asString)); + Assert.Equal(asBlob.GetHashCode(), comparer.GetHashCode(asBlob)); + } + } + } + + [Fact] + public void BinaryComparer_AgreesWithDefaultOnWellFormedText() + { + var binary = RedisValue.EqualityComparer.Binary; + foreach (var blob in EquivalenceBlobs()) + { + RedisValue asBlob = blob; + foreach (var s in EquivalenceStrings()) + { + // The two readings may differ exactly where UTF8 does not round-trip, and either side can be + // the culprit: a string holding an unpaired surrogate, or a blob that is not canonical UTF8 + // (0xFF decodes to U+FFFD but re-encodes to EF BF BD). Everywhere else they must agree. + if (Encoding.UTF8.GetString(Encoding.UTF8.GetBytes(s)) != s) continue; + if (!Encoding.UTF8.GetBytes(Encoding.UTF8.GetString(blob)).AsSpan().SequenceEqual(blob)) continue; + + RedisValue asString = s; + Assert.True( + (asString == asBlob) == binary.Equals(asString, asBlob), + $"'{Escape(s)}' vs {BitConverter.ToString(blob)}"); + } + } + } + + [Fact] + public void BinaryComparer_DivergesOnlyWhereUtf8DoesNotRoundTrip() + { + var binary = RedisValue.EqualityComparer.Binary; + + // a lone surrogate encodes to the same bytes as U+FFFD, so Binary calls them equal and the default + // does not - this is the documented divergence, pinned so it cannot drift silently + RedisValue loneSurrogate = new string([(char)0xD800]); + RedisValue replacement = "�"; + + Assert.False(loneSurrogate == replacement); + Assert.True(binary.Equals(loneSurrogate, replacement)); + + // and Binary stays self-consistent about it: equal means same hash + Assert.Equal(binary.GetHashCode(loneSurrogate), binary.GetHashCode(replacement)); + + // the other direction, where the *blob* is not canonical UTF8: 0xFF decodes to U+FFFD, so the default + // calls it equal to that text, while Binary sees FF against EF BF BD and does not + RedisValue invalidBlob = new byte[] { 0xFF }; + Assert.True(replacement == invalidBlob); + Assert.False(binary.Equals(replacement, invalidBlob)); + } + + [Fact] + public void BinaryComparer_EqualValuesShareHashCodes() + { + var binary = RedisValue.EqualityComparer.Binary; + foreach (var blob in EquivalenceBlobs()) + { + RedisValue asBlob = blob; + RedisValue asSegmented = blob.Length >= 2 ? Segmented(blob, blob.Length / 2) : asBlob; + + foreach (var s in EquivalenceStrings()) + { + RedisValue asString = s; + if (binary.Equals(asString, asBlob)) + { + Assert.True( + binary.GetHashCode(asString) == binary.GetHashCode(asBlob), + $"equal under Binary must share a hash: '{Escape(s)}' vs {BitConverter.ToString(blob)}"); + } + } + + // representation must not matter: the same bytes, contiguous or segmented + Assert.True(binary.Equals(asBlob, asSegmented)); + Assert.Equal(binary.GetHashCode(asBlob), binary.GetHashCode(asSegmented)); + } + } + + [Theory] + [InlineData("abc")] + [InlineData("42")] + public void Comparers_UntypedApiAcceptsWhateverRedisValueAccepts(string value) + { + foreach (var comparer in new[] { RedisValue.EqualityComparer.Default, RedisValue.EqualityComparer.Binary }) + { + var untyped = (IEqualityComparer)comparer; + object asString = value; + object asBytes = Encoding.UTF8.GetBytes(value); + object asRedisValue = (RedisValue)value; + + Assert.True(untyped.Equals(asString, asBytes)); + Assert.True(untyped.Equals(asString, asRedisValue)); + Assert.True(untyped.Equals(asBytes, asRedisValue)); + + Assert.Equal(untyped.GetHashCode(asString), untyped.GetHashCode(asBytes)); + Assert.Equal(untyped.GetHashCode(asString), untyped.GetHashCode(asRedisValue)); + + // something it cannot read is not equal to anything, but is still reflexive and stable + object unreadable = new object(); + Assert.False(untyped.Equals(unreadable, asString)); + Assert.True(untyped.Equals(unreadable, unreadable)); + Assert.Equal(unreadable.GetHashCode(), untyped.GetHashCode(unreadable)); + } + } + + // The chunked encode in Binary works 512 chars at a time, so anything shorter than that never reaches a + // chunk boundary - and every string in the corpus above is well under it. These put the awkward shapes + // exactly where the seam falls. + [Fact] + public void BinaryComparer_HandlesSurrogatesAtTheChunkBoundary() + { + var binary = RedisValue.EqualityComparer.Binary; + var pair = new string([(char)0xD83D, (char)0xDE00]); // U+1F600 + var loneHigh = new string([(char)0xD800]); + + foreach (var s in new[] + { + // a pair straddling the seam, behind enough three-byte characters that taking one *more* char + // would also overflow the encode buffer + new string('你', 511) + pair + "x", + + // a lone high surrogate at the seam, immediately followed by a real pair: extending the chunk + // would step onto the pair and split that instead + new string('a', 511) + loneHigh + pair + "x", + + // the seam landing inside a pair from the other side + new string('a', 510) + pair + pair + "x", + + // and a plain long value, so the multi-chunk path itself is covered + new string('a', 2000), + }) + { + RedisValue asString = s; + RedisValue asBlob = Encoding.UTF8.GetBytes(s); + Assert.True(binary.Equals(asString, asBlob), $"length {s.Length}"); + Assert.True(binary.Equals(asBlob, asString), $"length {s.Length} (reversed)"); + Assert.Equal(binary.GetHashCode(asString), binary.GetHashCode(asBlob)); + } + } + + [Fact] + public void BinaryComparer_DivergesOnNumericText() + { + // RedisValue equality runs Simplify() first, so text that parses to the same number is equal however + // it was spelled. Binary compares the bytes, so it is not - which is the right answer for a byte + // reading, and matches how the server identifies members, but it is a divergence worth pinning. + var binary = RedisValue.EqualityComparer.Binary; + foreach (var (a, b) in new[] { ("1.0", "1.00"), ("1", "1.0"), ("0", "-0.0"), ("1e2", "100") }) + { + RedisValue x = a, y = b; + Assert.True(x == y, $"'{a}' == '{b}' under the default rules"); + Assert.False(binary.Equals(x, y), $"'{a}' vs '{b}' under Binary"); + } + + // identical text still agrees, of course + Assert.True(binary.Equals((RedisValue)"42", (RedisValue)"42")); + } + + [Fact] + public void BinaryComparer_DoesNotDependOnHowAValueIsStored() + { + // Binary reads the UTF8 form of both sides, so the same text compares equal to itself however it + // arrived. The default rules do not manage this for every input - see #3233, where a string and the + // bytes of that same string can simplify differently - so this is a property worth holding onto. + var binary = RedisValue.EqualityComparer.Binary; + foreach (var text in new[] { "1,000", "(5)", " 5 ", "1,0,0,0", "1e2", "42", "hello", "" }) + { + RedisValue asString = text; + RedisValue asBlob = Encoding.UTF8.GetBytes(text); + + Assert.True(binary.Equals(asString, asBlob), $"'{text}' across storage"); + Assert.Equal(binary.GetHashCode(asString), binary.GetHashCode(asBlob)); + } + } + + [Fact] + public void BinaryComparer_WorksAsADictionaryComparer() + { + var dictionary = new Dictionary(RedisValue.EqualityComparer.Binary) + { + { "alpha", "one" }, + { Encoding.UTF8.GetBytes("beta"), "two" }, + }; + + Assert.Equal("one", dictionary[Encoding.UTF8.GetBytes("alpha")]); + Assert.Equal("two", dictionary["beta"]); + Assert.False(dictionary.ContainsKey("gamma")); + } }