From 3d4fea776cac3becadad5c1475f8d20614f539ad Mon Sep 17 00:00:00 2001 From: hanna Date: Sun, 12 Jul 2026 17:25:43 +0300 Subject: [PATCH 1/8] GH-50481: [C++] Fix CSV reader mis-parsing rows with an embedded NUL byte SSE42Filter::Matches used _mm_cmpistrc, an implicit-length SSE4.2 string-compare intrinsic that treats 0x00 as a terminator. Since it scans 8 raw CSV bytes at a time, a NUL byte embedded in a field could hide a real delimiter/quote/newline sharing the same 8-byte word, causing RunBulkFilter to blindly bulk-copy the word and silently mis-split the row. Switch to the explicit-length _mm_cmpestrc, passing the true lengths of both operands (sizeof(WordType) for the data word, sizeof(BulkFilterType) for the filter) instead of relying on NUL-termination. Verified locally: reverting just this fix reproduces the exact predicted failure ("Expected 64 columns, got 1") in the added regression test; with the fix, that test and the rest of the CSV test suite (272 tests) pass. This fix and its regression test were AI-generated (Claude), under human review and local verification. Co-Authored-By: Claude --- cpp/src/arrow/csv/lexing_internal.h | 7 ++++-- cpp/src/arrow/csv/parser_test.cc | 39 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index b1da12750ac5..b919c3833393 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -133,8 +133,11 @@ class SSE42Filter { explicit SSE42Filter(const ParseOptions& options) : filter_(MakeFilter(options)) {} bool Matches(WordType w) const { - // Look up every byte in `w` in the SIMD filter. - return _mm_cmpistrc(_mm_set1_epi64x(w), filter_, + // Look up every byte in `w` in the SIMD filter. Use the explicit-length + // comparison since `w` may contain an embedded NUL byte, which the + // implicit-length _mm_cmpistrc would otherwise treat as a terminator. + return _mm_cmpestrc(_mm_set1_epi64x(w), sizeof(WordType), filter_, + sizeof(BulkFilterType), _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY); } diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index abe57dd2e89c..e809b8374cd0 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -936,5 +936,44 @@ TEST(BlockParser, RowNumberAppendedToError) { } } +TEST(BlockParser, EmbeddedNulInQuotedFieldAfterBulkFilterActivates) { + // Regression test for GH-50481. Once the running average value length + // crosses the bulk filter's activation threshold, a NUL byte embedded in a + // quoted field could hide the real closing quote if both landed in the + // same 8-byte SIMD word, causing the parser to keep consuming subsequent + // bytes as if still inside the quoted field. + constexpr int32_t num_cols = 64; + constexpr int32_t num_filler_rows = 512; // = kTargetChunkSize / num_cols + + std::string csv; + for (int32_t r = 0; r < num_filler_rows; ++r) { + for (int32_t c = 0; c < num_cols; ++c) { + if (c) csv += ','; + // 12 bytes/value, well above the bulk filter's 10-bytes/value + // activation threshold. + csv += "xxxxxxxxxxxx"; + } + csv += '\n'; + } + // This row's first field carries a real embedded NUL byte immediately + // before its closing quote - the byte pattern a misaligned + // implicit-length SIMD scan can hide. + csv += "\"abc"; + csv += '\0'; + csv += "def\""; + for (int32_t c = 1; c < num_cols; ++c) { + csv += ",xxxxxxxxxxxx"; + } + csv += '\n'; + + BlockParser parser(ParseOptions::Defaults()); + AssertParseFinal(parser, csv); + ASSERT_EQ(parser.num_rows(), num_filler_rows + 1); + + std::vector last_row; + GetLastRow(parser, &last_row); + ASSERT_EQ(last_row[0], std::string("abc\0def", 7)); +} + } // namespace csv } // namespace arrow From 576461bcfd8b78c7c091350fd8a707844ea2c150 Mon Sep 17 00:00:00 2001 From: Hanna Weissberg <93118219+HannaWeissberg@users.noreply.github.com> Date: Sun, 12 Jul 2026 17:34:05 +0300 Subject: [PATCH 2/8] Adding int casting to sizeof() Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cpp/src/arrow/csv/lexing_internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index b919c3833393..f23472fe629b 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -136,8 +136,8 @@ class SSE42Filter { // Look up every byte in `w` in the SIMD filter. Use the explicit-length // comparison since `w` may contain an embedded NUL byte, which the // implicit-length _mm_cmpistrc would otherwise treat as a terminator. - return _mm_cmpestrc(_mm_set1_epi64x(w), sizeof(WordType), filter_, - sizeof(BulkFilterType), + return _mm_cmpestrc(_mm_set1_epi64x(w), static_cast(sizeof(WordType)), filter_, + static_cast(sizeof(BulkFilterType)), _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY); } From 4b2dca0e166f1b9491d6e18c03b166837da2292e Mon Sep 17 00:00:00 2001 From: Hanna Weissberg <93118219+HannaWeissberg@users.noreply.github.com> Date: Sun, 12 Jul 2026 17:41:32 +0300 Subject: [PATCH 3/8] Adding full size assert check --- cpp/src/arrow/csv/parser_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index e809b8374cd0..129498572e62 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -972,6 +972,7 @@ TEST(BlockParser, EmbeddedNulInQuotedFieldAfterBulkFilterActivates) { std::vector last_row; GetLastRow(parser, &last_row); + ASSERT_EQ(last_row.size(), static_cast(num_cols)); ASSERT_EQ(last_row[0], std::string("abc\0def", 7)); } From 92cb521393659dbde7579d744c7b1fea0f33a938 Mon Sep 17 00:00:00 2001 From: Hanna Weissberg <93118219+HannaWeissberg@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:10:46 +0300 Subject: [PATCH 4/8] Fix test Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- cpp/src/arrow/csv/parser_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index 129498572e62..0abe50f484c0 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -966,7 +966,7 @@ TEST(BlockParser, EmbeddedNulInQuotedFieldAfterBulkFilterActivates) { } csv += '\n'; - BlockParser parser(ParseOptions::Defaults()); + BlockParser parser(ParseOptions::Defaults(), /*num_cols=*/num_cols); AssertParseFinal(parser, csv); ASSERT_EQ(parser.num_rows(), num_filler_rows + 1); From 37c4eecbbb77c4ced17d205e69dd3a3c19f37f85 Mon Sep 17 00:00:00 2001 From: hanna Date: Wed, 22 Jul 2026 13:02:58 +0300 Subject: [PATCH 5/8] GH-50481: [C++] Disable bulk filter instead of switching SIMD compare @ursabot's benchmarks showed the previous _mm_cmpestrc fix caused 10-73% throughput regressions on several CSV parsing/chunking benchmarks (e.g. ChunkCSVVehiclesExample -72.9%), since the explicit-length compare is measurably slower than the implicit-length one it replaced. Per pitrou's suggestion, revert SSE42Filter::Matches to the original _mm_cmpistrc, and instead disable the bulk filter entirely for any block containing a NUL byte: BlockParserImpl now scans each block once via memchr (block_has_nul_), gating use_bulk_filter_ on it being unset. This keeps the fast path's cost paid only on NUL-free blocks, which is the common case, while blocks with an embedded NUL fall back to the already-correct per-byte path. This change and its test updates were AI-generated (Claude), under human review and local verification. Co-Authored-By: Claude --- cpp/src/arrow/csv/lexing_internal.h | 7 ++--- cpp/src/arrow/csv/parser.cc | 20 +++++++++++--- cpp/src/arrow/csv/parser_test.cc | 41 ++++++++++++++++++----------- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index f23472fe629b..b1da12750ac5 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -133,11 +133,8 @@ class SSE42Filter { explicit SSE42Filter(const ParseOptions& options) : filter_(MakeFilter(options)) {} bool Matches(WordType w) const { - // Look up every byte in `w` in the SIMD filter. Use the explicit-length - // comparison since `w` may contain an embedded NUL byte, which the - // implicit-length _mm_cmpistrc would otherwise treat as a terminator. - return _mm_cmpestrc(_mm_set1_epi64x(w), static_cast(sizeof(WordType)), filter_, - static_cast(sizeof(BulkFilterType)), + // Look up every byte in `w` in the SIMD filter. + return _mm_cmpistrc(_mm_set1_epi64x(w), filter_, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY); } diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc index 54dc54ae7e82..c0e9cca083d8 100644 --- a/cpp/src/arrow/csv/parser.cc +++ b/cpp/src/arrow/csv/parser.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -533,12 +534,13 @@ class BlockParserImpl { } if (batch_.num_rows_ > start_num_rows && batch_.num_cols_ > 0) { - // Use bulk filter only if average value length is >= 10 bytes, - // as the bulk filter has a fixed cost that isn't compensated - // when values are too short. + // Use bulk filter only if average value length is >= 10 bytes + // (its fixed cost isn't compensated for short values), and the block + // has no embedded NUL bytes (see block_has_nul_). const int64_t bulk_filter_threshold = static_cast(batch_.num_cols_) * (batch_.num_rows_ - start_num_rows) * 10; - use_bulk_filter_ = (data - *out_data) > bulk_filter_threshold; + use_bulk_filter_ = + !block_has_nul_ && (data - *out_data) > bulk_filter_threshold; } // Append new buffers and update size @@ -561,8 +563,17 @@ class BlockParserImpl { values_size_ = 0; size_t total_view_length = 0; + block_has_nul_ = false; for (const auto& view : views) { total_view_length += view.length(); +#if defined(ARROW_HAVE_SSE4_2) && (defined(__x86_64__) || defined(_M_X64)) + // Only the SSE4.2 bulk filter is affected by embedded NUL bytes, + // so only scan for them when that filter is available. + if (!block_has_nul_ && !view.empty() && + memchr(view.data(), '\0', view.length()) != nullptr) { + block_has_nul_ = true; + } +#endif } if (total_view_length > std::numeric_limits::max()) { return Status::Invalid("CSV block too large"); @@ -691,6 +702,7 @@ class BlockParserImpl { int32_t max_num_rows_; bool use_bulk_filter_ = false; + bool block_has_nul_ = false; // Unparsed data size int32_t values_size_; diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index 0abe50f484c0..bbb41c854f1f 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -936,37 +936,44 @@ TEST(BlockParser, RowNumberAppendedToError) { } } -TEST(BlockParser, EmbeddedNulInQuotedFieldAfterBulkFilterActivates) { - // Regression test for GH-50481. Once the running average value length - // crosses the bulk filter's activation threshold, a NUL byte embedded in a - // quoted field could hide the real closing quote if both landed in the - // same 8-byte SIMD word, causing the parser to keep consuming subsequent - // bytes as if still inside the quoted field. +TEST(BlockParser, EmbeddedNulBytesDisableBulkFilter) { + // Regression test for GH-50481: the bulk filter's implicit-length SIMD + // compare can miss a delimiter sharing a word with an embedded NUL byte. + // The fix disables the bulk filter for any block containing a NUL, so + // every cell here carries one. num_cols is explicit so the bulk filter + // isn't delayed by the single-row column-count-inference parse path. constexpr int32_t num_cols = 64; - constexpr int32_t num_filler_rows = 512; // = kTargetChunkSize / num_cols + // Each ParseChunk call is capped at ~512 rows for num_cols == 64 (from + // parser.cc's private kTargetChunkSize). Use 4x that margin so the filler + // block still spans multiple calls -- keeping the bulk filter armed + // before the trigger row -- even if that constant changes. + constexpr int32_t num_filler_rows = 4 * 512; + + // 12 bytes/value, above the bulk filter's 10-byte threshold; the + // trailing NUL lands outside the first SIMD word, so it's harmless here. + std::string filler_cell = "xxxxxxxxxxx"; + filler_cell += '\0'; std::string csv; for (int32_t r = 0; r < num_filler_rows; ++r) { for (int32_t c = 0; c < num_cols; ++c) { if (c) csv += ','; - // 12 bytes/value, well above the bulk filter's 10-bytes/value - // activation threshold. - csv += "xxxxxxxxxxxx"; + csv += filler_cell; } csv += '\n'; } - // This row's first field carries a real embedded NUL byte immediately - // before its closing quote - the byte pattern a misaligned - // implicit-length SIMD scan can hide. + // Embeds a NUL right before the closing quote - the pattern a + // misaligned SIMD scan can hide. csv += "\"abc"; csv += '\0'; csv += "def\""; for (int32_t c = 1; c < num_cols; ++c) { - csv += ",xxxxxxxxxxxx"; + csv += ','; + csv += filler_cell; } csv += '\n'; - BlockParser parser(ParseOptions::Defaults(), /*num_cols=*/num_cols); + BlockParser parser(ParseOptions::Defaults(), num_cols, /*first_row=*/0); AssertParseFinal(parser, csv); ASSERT_EQ(parser.num_rows(), num_filler_rows + 1); @@ -974,6 +981,10 @@ TEST(BlockParser, EmbeddedNulInQuotedFieldAfterBulkFilterActivates) { GetLastRow(parser, &last_row); ASSERT_EQ(last_row.size(), static_cast(num_cols)); ASSERT_EQ(last_row[0], std::string("abc\0def", 7)); + // The row's other NUL-bearing fields must come through unmangled too. + for (size_t c = 1; c < last_row.size(); ++c) { + ASSERT_EQ(last_row[c], filler_cell) << "column " << c; + } } } // namespace csv From 912b9ae19485f9b1784114f8adb05006e76fe2db Mon Sep 17 00:00:00 2001 From: hanna Date: Wed, 22 Jul 2026 14:38:13 +0300 Subject: [PATCH 6/8] GH-50481: [C++] Fix chunker.cc bulk filter and use_bulk_filter_ staleness use_bulk_filter_ could stay set across Parse() calls, letting a new block's own NUL slip through. chunker.cc's Lexer has a separate bulk filter untouched by the parser.cc fix. Fix both, and factor the NUL check into CanUseOnBlock() on each filter class. Co-Authored-By: Claude --- cpp/src/arrow/csv/chunker.cc | 3 +++ cpp/src/arrow/csv/chunker_test.cc | 20 ++++++++++++++++++++ cpp/src/arrow/csv/lexing_internal.h | 17 +++++++++++++++++ cpp/src/arrow/csv/parser.cc | 15 ++++++--------- cpp/src/arrow/csv/parser_test.cc | 21 +++++++-------------- 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/cpp/src/arrow/csv/chunker.cc b/cpp/src/arrow/csv/chunker.cc index 2ba097717691..1cf6919383e2 100644 --- a/cpp/src/arrow/csv/chunker.cc +++ b/cpp/src/arrow/csv/chunker.cc @@ -56,6 +56,9 @@ class Lexer { // Decide whether it's worth using a bulk filter over the given data area bool ShouldUseBulkFilter(const char* data, const char* data_end) { + if (!bulk_filter_.CanUseOnBlock(std::string_view(data, static_cast(data_end - data)))) { + return false; + } constexpr int32_t kWordSize = static_cast(sizeof(BulkWordType)); // Only probe the 32 first words and assume they are representative of the rest diff --git a/cpp/src/arrow/csv/chunker_test.cc b/cpp/src/arrow/csv/chunker_test.cc index 7b087da21d1a..40aace314b26 100644 --- a/cpp/src/arrow/csv/chunker_test.cc +++ b/cpp/src/arrow/csv/chunker_test.cc @@ -341,6 +341,26 @@ TEST_P(BaseChunkerTest, EscapingAndQuoting) { } } +TEST_P(BaseChunkerTest, EmbeddedNulBytesDisableBulkFilter) { + // Regression test for GH-50481 in Lexer's own bulk filter (used for + // chunking, separately from BlockParser's). + // + // Lead-in with no structural bytes so ShouldUseBulkFilter's probe of the + // first 256 bytes decides to use the bulk filter here. + std::string lead_in(300, 'x'); + // NUL right before the closing quote: the misaligned-SIMD-scan trigger. + std::string trigger_field = "abc"; + trigger_field += '\0'; + trigger_field += "def"; + std::string first_row = lead_in + ",\"" + trigger_field + "\",tail\n"; + // No trailing newline on the second row, so Process() must split it out + // as the partial remainder -- proving the first row's boundary was found. + std::string csv = first_row + "next"; + + MakeChunker(); + AssertChunkSize(*chunker_, csv, static_cast(first_row.size())); +} + TEST_P(BaseChunkerTest, ParseSkip) { { auto csv = MakeCSVData({"ab,c,\n", "def,,gh\n", ",ij,kl\n"}); diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index b1da12750ac5..f2efa8ea7e5d 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include #include "arrow/csv/options.h" #include "arrow/util/simd.h" @@ -45,6 +47,10 @@ class BaseBloomFilter { public: explicit BaseBloomFilter(const ParseOptions& options) : filter_(MakeFilter(options)) {} + // Bloom filters match bytes individually and have no implicit-length + // scanning, so an embedded NUL byte doesn't affect their correctness. + bool CanUseOnBlock(std::string_view) const { return true; } + protected: using FilterType = uint64_t; // 63 for uint64_t @@ -138,6 +144,13 @@ class SSE42Filter { _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY); } + // _mm_cmpistrc is an implicit-length compare: it treats a NUL byte as a + // terminator and can miss a real delimiter/quote/newline sharing an 8-byte + // word with one. Never use this filter on a block that contains a NUL. + bool CanUseOnBlock(std::string_view data) const { + return data.empty() || std::memchr(data.data(), '\0', data.size()) == nullptr; + } + protected: using BulkFilterType = __m128i; @@ -190,6 +203,10 @@ class NeonFilter { return r != 0; } + // NEON compares each byte independently (no implicit-length scanning), so + // an embedded NUL byte doesn't affect correctness here. + bool CanUseOnBlock(std::string_view) const { return true; } + private: const uint8x8_t delim_, quote_, escape_; }; diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc index c0e9cca083d8..bed94d197038 100644 --- a/cpp/src/arrow/csv/parser.cc +++ b/cpp/src/arrow/csv/parser.cc @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -539,8 +538,7 @@ class BlockParserImpl { // has no embedded NUL bytes (see block_has_nul_). const int64_t bulk_filter_threshold = static_cast(batch_.num_cols_) * (batch_.num_rows_ - start_num_rows) * 10; - use_bulk_filter_ = - !block_has_nul_ && (data - *out_data) > bulk_filter_threshold; + use_bulk_filter_ = !block_has_nul_ && (data - *out_data) > bulk_filter_threshold; } // Append new buffers and update size @@ -566,14 +564,13 @@ class BlockParserImpl { block_has_nul_ = false; for (const auto& view : views) { total_view_length += view.length(); -#if defined(ARROW_HAVE_SSE4_2) && (defined(__x86_64__) || defined(_M_X64)) - // Only the SSE4.2 bulk filter is affected by embedded NUL bytes, - // so only scan for them when that filter is available. - if (!block_has_nul_ && !view.empty() && - memchr(view.data(), '\0', view.length()) != nullptr) { + if (!block_has_nul_ && !bulk_filter.CanUseOnBlock(view)) { block_has_nul_ = true; } -#endif + } + if (block_has_nul_) { + // Clear a bulk filter left on by an earlier NUL-free block. + use_bulk_filter_ = false; } if (total_view_length > std::numeric_limits::max()) { return Status::Invalid("CSV block too large"); diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index bbb41c854f1f..719f13d65f9f 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -937,20 +937,14 @@ TEST(BlockParser, RowNumberAppendedToError) { } TEST(BlockParser, EmbeddedNulBytesDisableBulkFilter) { - // Regression test for GH-50481: the bulk filter's implicit-length SIMD - // compare can miss a delimiter sharing a word with an embedded NUL byte. - // The fix disables the bulk filter for any block containing a NUL, so - // every cell here carries one. num_cols is explicit so the bulk filter - // isn't delayed by the single-row column-count-inference parse path. + // Regression test for GH-50481: disables the bulk filter for any block + // with an embedded NUL, so every cell here carries one. constexpr int32_t num_cols = 64; - // Each ParseChunk call is capped at ~512 rows for num_cols == 64 (from - // parser.cc's private kTargetChunkSize). Use 4x that margin so the filler - // block still spans multiple calls -- keeping the bulk filter armed - // before the trigger row -- even if that constant changes. + // 4x the ~512-row ParseChunk cap for num_cols == 64, so the filler block + // spans multiple calls even if that internal constant changes. constexpr int32_t num_filler_rows = 4 * 512; - // 12 bytes/value, above the bulk filter's 10-byte threshold; the - // trailing NUL lands outside the first SIMD word, so it's harmless here. + // 12 bytes/value, above the bulk filter's activation threshold. std::string filler_cell = "xxxxxxxxxxx"; filler_cell += '\0'; @@ -962,8 +956,7 @@ TEST(BlockParser, EmbeddedNulBytesDisableBulkFilter) { } csv += '\n'; } - // Embeds a NUL right before the closing quote - the pattern a - // misaligned SIMD scan can hide. + // NUL right before the closing quote: the misaligned-SIMD-scan trigger. csv += "\"abc"; csv += '\0'; csv += "def\""; @@ -981,7 +974,7 @@ TEST(BlockParser, EmbeddedNulBytesDisableBulkFilter) { GetLastRow(parser, &last_row); ASSERT_EQ(last_row.size(), static_cast(num_cols)); ASSERT_EQ(last_row[0], std::string("abc\0def", 7)); - // The row's other NUL-bearing fields must come through unmangled too. + // Other NUL-bearing fields in the row must come through unmangled too. for (size_t c = 1; c < last_row.size(); ++c) { ASSERT_EQ(last_row[c], filler_cell) << "column " << c; } From 862cba9d83de346e41c107a46dea020c954b66e4 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 22 Jul 2026 14:25:59 +0200 Subject: [PATCH 7/8] Enhance comment --- cpp/src/arrow/csv/lexing_internal.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index f2efa8ea7e5d..b45af7a370d4 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -147,6 +147,10 @@ class SSE42Filter { // _mm_cmpistrc is an implicit-length compare: it treats a NUL byte as a // terminator and can miss a real delimiter/quote/newline sharing an 8-byte // word with one. Never use this filter on a block that contains a NUL. + // We could instead use the explicit-length compare _mm_cmpestrc but + // it comes with a massive performance cost on some CPUs + // (some benchmarks were measured to be twice slower in + // https://github.com/apache/arrow/pull/50483). bool CanUseOnBlock(std::string_view data) const { return data.empty() || std::memchr(data.data(), '\0', data.size()) == nullptr; } From 407eec08c225986ae6a53b6829c45c4289ef02a2 Mon Sep 17 00:00:00 2001 From: hanna Date: Wed, 22 Jul 2026 16:15:00 +0300 Subject: [PATCH 8/8] GH-50481: [C++] Fix clang-format lint failure in chunker.cc Co-Authored-By: Claude --- cpp/src/arrow/csv/chunker.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/csv/chunker.cc b/cpp/src/arrow/csv/chunker.cc index 1cf6919383e2..55019e9dab33 100644 --- a/cpp/src/arrow/csv/chunker.cc +++ b/cpp/src/arrow/csv/chunker.cc @@ -56,7 +56,8 @@ class Lexer { // Decide whether it's worth using a bulk filter over the given data area bool ShouldUseBulkFilter(const char* data, const char* data_end) { - if (!bulk_filter_.CanUseOnBlock(std::string_view(data, static_cast(data_end - data)))) { + if (!bulk_filter_.CanUseOnBlock( + std::string_view(data, static_cast(data_end - data)))) { return false; } constexpr int32_t kWordSize = static_cast(sizeof(BulkWordType));