Compare strings against blobs without decoding them into a string

src/StackExchange.Redis/RedisValue.cs

@@ -11,6 +11,9 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
+#if NET
+using System.Text.Unicode;
+#endif
 using RESPite;
 
 namespace StackExchange.Redis
@@ -477,10 +480,157 @@ internal string RawString()
             // memory / short-blob / sequence) compare by raw bytes in any combination
             if (IsBlob(xType) && IsBlob(yType)) return BlobSequenceEqual(x, y);
 
-            // otherwise (anything involving a string), compare as strings
-            return (string?)x == (string?)y;
+            // otherwise: exactly one side is a string and the other is byte-backed (everything else has
+            // been dealt with above). The relation is unchanged - "the blob, read as UTF-8 text, equals the
+            // string" - but it is answered without materialising that text: see StringEqualsBlob.
+            return xType == StorageType.String
+                ? StringEqualsBlob(x.RawString(), in y)
+                : StringEqualsBlob(y.RawString(), in x);
         }
 
+        /// <summary>
+        /// Compares a string against the UTF-8 text of a byte-backed value, without decoding it into a string.
+        /// </summary>
+        /// <remarks>
+        /// <para>
+        /// Identical in meaning to <c>s == (string)blob</c>, deliberately: equality here is defined by
+        /// decoding the blob, and <see cref="GetHashCode(RedisValue)"/> hashes the same decoded form. Comparing
+        /// the other way around - encoding the string and matching bytes - is *not* the same relation, because
+        /// UTF-8 does not round-trip strings holding unpaired surrogates, and it would put equal values in
+        /// different hash buckets.
+        /// </para>
+        /// <para>
+        /// The decode runs a chunk at a time into a small stack buffer, so a long value costs no allocation
+        /// (the previous form allocated a transient string, which for values over ~42k chars landed on the
+        /// large object heap), and a mismatch stops at the chunk that contains it rather than after decoding
+        /// the whole payload.
+        /// </para>
+        /// </remarks>
+        private static bool StringEqualsBlob(string s, in RedisValue blob)
+        {
+            // Bounds, before touching any content: UTF-8 never yields more chars than it has bytes, and never
+            // spends more than 3 bytes per char (4-byte sequences produce 2 chars, i.e. 2 bytes per char).
+            int byteLength = blob.BlobLength;
+            if (s.Length > byteLength || (long)s.Length * 3 < byteLength) return false;
+
+#if NET
+            if (blob.Type == StorageType.Sequence)
+            {
+                var seq = blob.RawSequence();
+                if (!seq.IsSingleSegment) return StringEqualsUtf8(s, seq);
+                return StringEqualsUtf8(s, seq.First.Span);
+            }
+            return StringEqualsUtf8(s, blob.UnsafeRawSpan(out _));
+#else
+            // no System.Text.Unicode.Utf8 on the down-level targets; keep the original behaviour there
+            return s == (string?)blob;
+#endif
+        }
+
+#if NET
+        /// <summary>Chars decoded per pass; small enough that the stack cost is irrelevant.</summary>
+        private const int CompareChunkChars = 128;
+
+        /// <summary>Longest UTF-8 sequence, i.e. the most that can be left pending at a segment boundary.</summary>
+        private const int MaxUtf8SequenceLength = 4;
+
+        private static bool StringEqualsUtf8(string s, scoped ReadOnlySpan<byte> utf8)
+        {
+            Span<char> chars = stackalloc char[CompareChunkChars];
+            int matched = 0;
+            while (!utf8.IsEmpty)
+            {
+                // isFinalBlock: false - a chunk boundary landing mid-sequence must be resumed on the next
+                // pass, not reported as invalid data
+                Utf8.ToUtf16(utf8, chars, out int bytesRead, out int charsWritten, replaceInvalidSequences: true, isFinalBlock: false);
+                if (bytesRead == 0 && charsWritten == 0)
+                {
+                    // what remains is a truncated sequence at the very end; decoding it as final is what
+                    // yields the replacement char that the string form would have carried
+                    Utf8.ToUtf16(utf8, chars, out bytesRead, out charsWritten, replaceInvalidSequences: true, isFinalBlock: true);
+                    // Defensive, and believed unreachable: a final-block decode of a non-empty input always
+                    // yields at least the replacement character. Kept so that a future change which makes it
+                    // reachable fails the comparison rather than spinning here forever.
+                    if (bytesRead == 0 && charsWritten == 0) return false;
+                }
+                if (!AdvanceMatch(s, chars.Slice(0, charsWritten), ref matched)) return false;
+                utf8 = utf8.Slice(bytesRead);
+            }
+            return matched == s.Length;
+        }
+
+        private static bool StringEqualsUtf8(string s, scoped in ReadOnlySequence<byte> utf8)
+        {
+            Span<char> chars = stackalloc char[CompareChunkChars];
+            Span<byte> pending = stackalloc byte[MaxUtf8SequenceLength];
+            int pendingLength = 0, matched = 0;
+
+            foreach (var segment in utf8)
+            {
+                var span = segment.Span;
+                while (!span.IsEmpty)
+                {
+                    if (pendingLength != 0)
+                    {
+                        // a sequence split across segments: top it up from this one and decode just that
+                        int take = Math.Min(MaxUtf8SequenceLength - pendingLength, span.Length);
+                        span.Slice(0, take).CopyTo(pending.Slice(pendingLength));
+                        int available = pendingLength + take;
+
+                        Utf8.ToUtf16(pending.Slice(0, available), chars, out int joinedBytes, out int joinedChars, replaceInvalidSequences: true, isFinalBlock: false);
+                        if (joinedBytes == 0)
+                        {
+                            // still short: absorb what we took and look to the next segment
+                            pendingLength = available;
+                            span = span.Slice(take);
+                            continue;
+                        }
+
+                        if (!AdvanceMatch(s, chars.Slice(0, joinedChars), ref matched)) return false;
+                        // The decoder consumed the carried prefix and possibly more, never less: `pending`
+                        // only ever holds a sequence that was incomplete but valid, and such a sequence is
+                        // consumed as a unit. Asserted because if that ever stopped holding, the slice below
+                        // would throw rather than quietly answer wrongly.
+                        Debug.Assert(joinedBytes >= pendingLength, "decoder consumed less than the carried prefix");
+                        span = span.Slice(joinedBytes - pendingLength); // give back the bytes we borrowed but did not use
+                        pendingLength = 0;
+                        continue;
+                    }
+
+                    Utf8.ToUtf16(span, chars, out int bytesRead, out int charsWritten, replaceInvalidSequences: true, isFinalBlock: false);
+                    if (bytesRead == 0 && charsWritten == 0)
+                    {
+                        // trailing partial sequence: carry it into the next segment
+                        span.CopyTo(pending);
+                        pendingLength = span.Length;
+                        break;
+                    }
+
+                    if (!AdvanceMatch(s, chars.Slice(0, charsWritten), ref matched)) return false;
+                    span = span.Slice(bytesRead);
+                }
+            }
+
+            if (pendingLength != 0)
+            {
+                // truncated at the end of the payload
+                Utf8.ToUtf16(pending.Slice(0, pendingLength), chars, out _, out int finalChars, replaceInvalidSequences: true, isFinalBlock: true);
+                if (!AdvanceMatch(s, chars.Slice(0, finalChars), ref matched)) return false;
+            }
+
+            return matched == s.Length;
+        }
+
+        /// <summary>Matches freshly decoded chars against the next part of the string; false ends the compare.</summary>
+        private static bool AdvanceMatch(string s, scoped ReadOnlySpan<char> decoded, ref int matched)
+        {
+            if (matched + decoded.Length > s.Length) return false;
+            if (!decoded.SequenceEqual(s.AsSpan(matched, decoded.Length))) return false;
+            matched += decoded.Length;
+            return true;
+        }
+#endif
+
         /// <summary>
         /// See <see cref="object.Equals(object)"/>.
         /// </summary>

tests/StackExchange.Redis.Benchmarks/RedisValueEqualityBenchmarks.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Buffers;
+using System.Text;
+using BenchmarkDotNet.Attributes;
+
+namespace StackExchange.Redis.Benchmarks
+{
+    /// <summary>
+    /// Sizes the cost of <see cref="RedisValue"/> equality where one side is a string and the other a blob.
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// That mixed case is the one with room to move: everything else already compares in place. The
+    /// DiffAtStart shape is the interesting one, since it separates the cost of comparing from the cost of
+    /// materialising - an implementation that decodes before it compares pays the same for a value that
+    /// differs in its first byte as for one that matches.
+    /// </para>
+    /// <para>
+    /// Note that the baseline here is <see cref="StringVsByteArray"/>, so the ratio column compares the other
+    /// shapes against the mixed case *within one build*. Comparing an implementation against its predecessor
+    /// means running this on both commits and lining the two tables up by hand; there is no in-run before and
+    /// after.
+    /// </para>
+    /// </remarks>
+    [Config(typeof(SlowConfig))]
+    public class RedisValueEqualityBenchmarks
+    {
+        public enum Shape
+        {
+            /// <summary>Both sides identical; the full payload has to be examined either way.</summary>
+            Equal,
+
+            /// <summary>Differs in the first byte: everything after it is wasted work.</summary>
+            DiffAtStart,
+
+            /// <summary>Differs in the last byte: the whole payload must be examined regardless.</summary>
+            DiffAtEnd,
+        }
+
+        [Params(16, 1024, 65536)]
+        public int Size { get; set; }
+
+        [Params(Shape.Equal, Shape.DiffAtStart, Shape.DiffAtEnd)]
+        public Shape Form { get; set; }
+
+        private RedisValue _string, _other, _byteArray, _sequence, _stringB;
+
+        private static string MakeString(int size, Shape form)
+        {
+            // deliberately non-numeric: anything numeric is reduced by Simplify() and never reaches the
+            // string/blob comparison at all
+            var chars = new char[size];
+            for (int i = 0; i < size; i++) chars[i] = (char)('a' + (i % 26));
+            var s = new string(chars);
+            return form switch
+            {
+                Shape.DiffAtStart => "Z" + s.Substring(1),
+                Shape.DiffAtEnd => s.Substring(0, size - 1) + "Z",
+                _ => s,
+            };
+        }
+
+        private static ReadOnlySequence<byte> AsSegmented(byte[] bytes)
+        {
+            // split in the middle, so the multi-segment path is exercised rather than the fast single-span one
+            int mid = bytes.Length / 2;
+            var first = new Segment(new ReadOnlyMemory<byte>(bytes, 0, mid), null);
+            var second = new Segment(new ReadOnlyMemory<byte>(bytes, mid, bytes.Length - mid), first);
+            return new ReadOnlySequence<byte>(first, 0, second, second.Memory.Length);
+        }
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            var baseline = MakeString(Size, Shape.Equal);
+            var variant = MakeString(Size, Form);
+
+            _string = baseline;
+            _stringB = variant;
+
+            var bytes = Encoding.UTF8.GetBytes(variant);
+            _byteArray = bytes;
+            _sequence = AsSegmented(bytes);
+            _other = Encoding.UTF8.GetBytes(baseline);
+        }
+
+        /// <summary>String vs byte[]: the mixed case, where one side has to be converted to meet the other.</summary>
+        [Benchmark(Baseline = true)]
+        public bool StringVsByteArray() => _string == _byteArray;
+
+        /// <summary>String vs a multi-segment blob - same mixed case, sequence-backed.</summary>
+        [Benchmark]
+        public bool StringVsSequence() => _string == _sequence;
+
+        /// <summary>Control: blob vs blob already compares by raw bytes, with no decode.</summary>
+        [Benchmark]
+        public bool BlobVsBlob() => _other == _byteArray;
+
+        /// <summary>Control: string vs string, the plain managed comparison.</summary>
+        [Benchmark]
+        public bool StringVsString() => _string == _stringB;
+
+        /// <summary>Hashing a blob: decodes the whole payload into a pooled char buffer.</summary>
+        [Benchmark]
+        public int HashBlob() => _byteArray.GetHashCode();
+
+        /// <summary>Hashing the equivalent string, for comparison.</summary>
+        [Benchmark]
+        public int HashString() => _string.GetHashCode();
+
+        private sealed class Segment : ReadOnlySequenceSegment<byte>
+        {
+            public Segment(ReadOnlyMemory<byte> value, Segment? head)
+            {
+                Memory = value;
+                if (head is not null)
+                {
+                    RunningIndex = head.RunningIndex + head.Memory.Length;
+                    head.Next = this;
+                }
+            }
+        }
+    }
+}