diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index a0b479a4eeaefa..4a57c83624bb43 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1393,6 +1393,10 @@ DEFINE_mInt64(snii_forced_spill_min_arena_bytes, "67108864"); // merge-compacted into one (bounds the k-way merge fan-in and its open fds; // every run is held open for the whole merge). 0 = uncapped. Default 64. DEFINE_mInt32(snii_spill_max_run_files_per_buffer, "64"); +// Sparse SNII norms: a norms section stores bytes only for the rows that carry a norm when that +// is smaller than one byte per row. Off writes the legacy dense layout older BEs read; readers +// accept both layouts either way. +DEFINE_mBool(enable_snii_sparse_norms, "true"); // dict path for chinese analyzer DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict"); // The kuromoji (Japanese) analyzer @@ -1408,10 +1412,11 @@ DEFINE_mBool(debug_inverted_index_compaction, "false"); DEFINE_mBool(inverted_index_ram_dir_enable, "true"); // wheather index by RAM directory when base compaction DEFINE_mBool(inverted_index_ram_dir_enable_when_base_compaction, "true"); -// Norms cost one byte per segment row, including rows that hold no value for the field. A segment -// holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on -// to leave norms out of every index on a variant path, whatever its "norms" property says; BM25 -// scoring (score()) on those indexes then fails. +// Norms cost one byte per segment row, including rows that hold no value for the field (SNII skips +// those rows while enable_snii_sparse_norms is on). A segment holds one index per variant path, so +// writing norms for them costs rows * paths bytes. Turn this on to leave norms out of every index +// on a variant path, whatever its "norms" property says; BM25 scoring (score()) on those indexes +// then fails. DEFINE_mBool(inverted_index_skip_norms_for_variant, "false"); // use num_broadcast_buffer blocks as buffer to do broadcast DEFINE_Int32(num_broadcast_buffer, "32"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 6bda64a14553a2..6e9376e77a1cc4 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1447,6 +1447,15 @@ DECLARE_mInt64(snii_forced_spill_min_arena_bytes); // run counts across ~100 concurrent writers can exhaust the BE nofile rlimit // ("Too many open files" at run reopen). 0 disables the cap. Default 64. DECLARE_mInt32(snii_spill_max_run_files_per_buffer); +// Lets SNII write BM25 norms in the sparse layout (section type kNormsSparse), which stores a +// norm only for the rows that carry one, whenever that is smaller than the dense layout's one +// byte per row; mostly NULL columns and VARIANT paths shrink the most. When off, every index +// written from then on -- load, compaction output, schema change, BUILD INDEX -- uses the legacy +// dense layout, which BEs without sparse-norms support can read: turn it off while such BEs may +// read newly written segments (a rolling upgrade, or before a downgrade). Segments already +// written keep their layout until they are rewritten. Read each time a logical index is +// finished. Reading sparse sections is always supported, whatever the value. +DECLARE_mBool(enable_snii_sparse_norms); // dict path for chinese analyzer DECLARE_String(inverted_index_dict_path); // The kuromoji (Japanese) analyzer @@ -1462,10 +1471,11 @@ DECLARE_mBool(debug_inverted_index_compaction); DECLARE_mBool(inverted_index_ram_dir_enable); // wheather index by RAM directory when base compaction DECLARE_mBool(inverted_index_ram_dir_enable_when_base_compaction); -// Norms cost one byte per segment row, including rows that hold no value for the field. A segment -// holds one index per variant path, so writing norms for them costs rows * paths bytes. Turn this on -// to leave norms out of every index on a variant path, whatever its "norms" property says; BM25 -// scoring (score()) on those indexes then fails. +// Norms cost one byte per segment row, including rows that hold no value for the field (SNII skips +// those rows while enable_snii_sparse_norms is on). A segment holds one index per variant path, so +// writing norms for them costs rows * paths bytes. Turn this on to leave norms out of every index +// on a variant path, whatever its "norms" property says; BM25 scoring (score()) on those indexes +// then fails. DECLARE_mBool(inverted_index_skip_norms_for_variant); // use num_broadcast_buffer blocks as buffer to do broadcast DECLARE_Int32(num_broadcast_buffer); diff --git a/be/src/storage/index/index_file_writer.cpp b/be/src/storage/index/index_file_writer.cpp index 923a39dda508f1..31469e86acc9a1 100644 --- a/be/src/storage/index/index_file_writer.cpp +++ b/be/src/storage/index/index_file_writer.cpp @@ -267,10 +267,12 @@ Status IndexFileWriter::add_snii_index(const TabletIndex* index_meta, uint32_t d input.config = index_config; input.doc_count = doc_count; input.null_docids = std::move(null_docids); + input.write_norms = options.write_norms; input.encoded_norms = std::move(options.encoded_norms); + input.null_docids_with_norms = std::move(options.null_docids_with_norms); input.term_source = term_buffer; input.mem_reporter = mem_reporter; - snii_resolve_index_write_params(options.is_direct_load, !input.encoded_norms.empty(), &input); + snii_resolve_index_write_params(options.is_direct_load, input.write_norms, &input); RETURN_IF_ERROR(_snii_compound_writer->add_logical_index(input)); ++_snii_index_count; return Status::OK(); diff --git a/be/src/storage/index/index_file_writer.h b/be/src/storage/index/index_file_writer.h index 3a0346b95d428a..be5429714c39eb 100644 --- a/be/src/storage/index/index_file_writer.h +++ b/be/src/storage/index/index_file_writer.h @@ -110,9 +110,13 @@ class IndexFileWriter { // the prx region compresses at snii_prx_zstd_level_direct_load; // compaction / schema change / ADD INDEX keep snii_prx_zstd_level. bool is_direct_load = false; - // One byte of BM25 norms per document; empty for keyword or positionless indexes. - // If nonempty, its size must equal doc_count, and postings retain frequencies for scoring. + // The index writes BM25 norms (analyzed indexes with positions, unless the norms + // policy turns them off); postings then retain frequencies for scoring. + bool write_norms = false; + // One encoded norm per document that carries one, in docid order: every document + // outside null_docids plus null_docids_with_norms. See SniiIndexInput. std::vector encoded_norms; + std::vector null_docids_with_norms; }; Status add_snii_index(const TabletIndex* index_meta, uint32_t doc_count, std::vector null_docids, diff --git a/be/src/storage/index/inverted/similarity/collection_statistics.cpp b/be/src/storage/index/inverted/similarity/collection_statistics.cpp index 06f88bacbb1689..bd8a3b2a693c87 100644 --- a/be/src/storage/index/inverted/similarity/collection_statistics.cpp +++ b/be/src/storage/index/inverted/similarity/collection_statistics.cpp @@ -17,6 +17,7 @@ #include "storage/index/inverted/similarity/collection_statistics.h" +#include #include #include #include @@ -41,7 +42,7 @@ namespace doris { namespace collection_statistics_detail { -Result resolve_snii_scoring_segment(uint64_t index_doc_count, +Result resolve_snii_scoring_segment(uint64_t indexed_doc_count, uint64_t sum_total_term_freq, bool has_positions, bool has_norms) { if (!has_positions || !has_norms) { @@ -51,7 +52,7 @@ Result resolve_snii_scoring_segment(uint64_t index_doc_ "\"false\" or, for a variant path, when inverted_index_skip_norms_for_variant is " "on")); } - return SniiScoringSegmentStats {.doc_count = index_doc_count, + return SniiScoringSegmentStats {.doc_count = indexed_doc_count, .token_count = sum_total_term_freq}; } @@ -133,10 +134,10 @@ Status CollectionStatistics::collect_full_collection( } oss << "]"; - oss << ", total_num_docs=" << _total_num_docs; - for (const auto& [ws_field_name, num_tokens] : _total_num_tokens) { + const auto num_docs = _total_num_docs.find(ws_field_name); oss << ", {field=" << StringHelper::to_string(ws_field_name) + << ", num_docs=" << (num_docs == _total_num_docs.end() ? 0 : num_docs->second) << ", num_tokens=" << num_tokens << ", terms=["; auto field_term_doc_freqs = _term_doc_freqs.find(ws_field_name); @@ -245,11 +246,19 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, return status; } auto logical_reader = std::move(logical_reader_result.value()); - const uint64_t segment_doc_count = logical_reader->stats().doc_count; + const auto& logical_stats = logical_reader->stats(); + const uint64_t segment_doc_count = logical_stats.doc_count; + // Every SNII writer stores indexed_doc_count = doc_count - null_count, and the + // metadata decoder rejects a stats block without it. + if (logical_stats.indexed_doc_count > segment_doc_count) { + return Status::Error( + "SNII indexed document count {} exceeds segment document count {}", + logical_stats.indexed_doc_count, segment_doc_count); + } RETURN_IF_ERROR(admit_snii_scoring_segment( - ws_field_name, segment_doc_count, logical_reader->stats().sum_total_term_freq, - logical_reader->has_positions(), logical_reader->has_norms(), - &segment_accumulator)); + ws_field_name, logical_stats.indexed_doc_count, + logical_stats.sum_total_term_freq, logical_reader->has_positions(), + logical_reader->has_norms(), &segment_accumulator)); ::doris::snii::reader::DictBlockCache dict_block_cache; for (const auto& logical_term_bytes : collect_info.unique_terms) { @@ -342,7 +351,9 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, } } - _total_num_docs += static_cast(total_segment_docs); + for (const auto& [ws_field_name, collect_info] : collect_infos) { + _total_num_docs[ws_field_name] += static_cast(total_segment_docs); + } _avg_dl_by_col.clear(); _idf_by_col_term.clear(); @@ -350,29 +361,25 @@ Status CollectionStatistics::process_segment(const RowsetSharedPtr& rowset, } Status CollectionStatistics::admit_snii_scoring_segment( - const std::wstring& field_name, uint64_t index_doc_count, uint64_t sum_total_term_freq, + const std::wstring& field_name, uint64_t indexed_doc_count, uint64_t sum_total_term_freq, bool has_positions, bool has_norms, SniiScoringSegmentAccumulator* segment_accumulator) { DORIS_CHECK(segment_accumulator != nullptr); auto segment_stats = collection_statistics_detail::resolve_snii_scoring_segment( - index_doc_count, sum_total_term_freq, has_positions, has_norms); + indexed_doc_count, sum_total_term_freq, has_positions, has_norms); if (!segment_stats.has_value()) { clear(); return segment_stats.error(); } - if (!segment_accumulator->token_counts.empty() && - segment_accumulator->doc_count != segment_stats->doc_count) { - clear(); - return Status::Error( - "SNII scoring fields in one segment have different document counts: {} and {}", - segment_accumulator->doc_count, segment_stats->doc_count); - } - segment_accumulator->doc_count = segment_stats->doc_count; + segment_accumulator->doc_counts[field_name] += segment_stats->doc_count; segment_accumulator->token_counts[field_name] += segment_stats->token_count; return Status::OK(); } void CollectionStatistics::commit_snii_scoring_segment( SniiScoringSegmentAccumulator&& segment_accumulator) { + for (const auto& [field_name, doc_count] : segment_accumulator.doc_counts) { + _total_num_docs[field_name] += doc_count; + } for (const auto& [field_name, token_count] : segment_accumulator.token_counts) { _total_num_tokens[field_name] += token_count; } @@ -381,13 +388,12 @@ void CollectionStatistics::commit_snii_scoring_segment( _term_doc_freqs[field_name][term] += doc_freq; } } - _total_num_docs += segment_accumulator.doc_count; _avg_dl_by_col.clear(); _idf_by_col_term.clear(); } void CollectionStatistics::clear() { - _total_num_docs = 0; + _total_num_docs.clear(); _total_num_tokens.clear(); _term_doc_freqs.clear(); _avg_dl_by_col.clear(); @@ -424,14 +430,15 @@ uint64_t CollectionStatistics::get_total_term_cnt_by_col(const std::wstring& luc return token_count->second; } -uint64_t CollectionStatistics::get_doc_num() const { - if (_total_num_docs == 0) { - throw Exception( - ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, - "Index statistics collection failed: No data available for SimilarityCollector"); +uint64_t CollectionStatistics::get_doc_num(const std::wstring& lucene_col_name) const { + const auto doc_count = _total_num_docs.find(lucene_col_name); + if (doc_count == _total_num_docs.end()) { + throw Exception(ErrorCode::INVERTED_INDEX_CLUCENE_ERROR, + "Index statistics collection failed: Not such column {}", + StringHelper::to_string(lucene_col_name)); } - return _total_num_docs; + return doc_count->second; } float CollectionStatistics::get_or_calculate_avg_dl(const std::wstring& lucene_col_name) { @@ -441,8 +448,12 @@ float CollectionStatistics::get_or_calculate_avg_dl(const std::wstring& lucene_c } const uint64_t total_term_cnt = get_total_term_cnt_by_col(lucene_col_name); - const uint64_t total_doc_cnt = get_doc_num(); - float avg_dl = total_doc_cnt > 0 ? float((double)total_term_cnt / (double)total_doc_cnt) : 0.0F; + // A field without documents has no tokens either, except for NULL ARRAY rows that kept + // their tokens (they sit in postings but not in the indexed count). Dividing by at least + // one keeps avgdl finite, and positive whenever a posting exists. + const uint64_t total_doc_cnt = std::max(get_doc_num(lucene_col_name), 1); + const auto avg_dl = static_cast(static_cast(total_term_cnt) / + static_cast(total_doc_cnt)); _avg_dl_by_col[lucene_col_name] = avg_dl; return avg_dl; } @@ -457,10 +468,13 @@ float CollectionStatistics::get_or_calculate_idf(const std::wstring& lucene_col_ } } - const uint64_t doc_num = get_doc_num(); const uint64_t doc_freq = get_term_doc_freq_by_col(lucene_col_name, term); - auto idf = (float)std::log(1 + ((double)doc_num - (double)doc_freq + (double)0.5) / - ((double)doc_freq + (double)0.5)); + // doc_freq never exceeds the document count, except on SNII fields whose NULL ARRAY rows + // kept tokens: those rows count in doc_freq but not in the indexed document count. Using + // the larger value keeps idf positive; it changes nothing for any other collection. + const uint64_t doc_num = std::max(get_doc_num(lucene_col_name), doc_freq); + auto idf = (float)std::log(1 + ((double)doc_num - (double)doc_freq + 0.5) / + ((double)doc_freq + 0.5)); _idf_by_col_term[lucene_col_name][term] = idf; return idf; } diff --git a/be/src/storage/index/inverted/similarity/collection_statistics.h b/be/src/storage/index/inverted/similarity/collection_statistics.h index 7e93498508324a..71c23dd2772e9a 100644 --- a/be/src/storage/index/inverted/similarity/collection_statistics.h +++ b/be/src/storage/index/inverted/similarity/collection_statistics.h @@ -71,7 +71,7 @@ class CollectionStatistics { private: struct SniiScoringSegmentAccumulator { - uint64_t doc_count = 0; + std::unordered_map doc_counts; std::unordered_map token_counts; std::unordered_map> term_doc_freqs; }; @@ -83,7 +83,7 @@ class CollectionStatistics { Status process_segment(const RowsetSharedPtr& rowset, const RowsetSegmentView& seg, const TabletSchema* tablet_schema, const CollectInfoMap& collect_infos, io::IOContext* io_ctx); - Status admit_snii_scoring_segment(const std::wstring& field_name, uint64_t index_doc_count, + Status admit_snii_scoring_segment(const std::wstring& field_name, uint64_t indexed_doc_count, uint64_t sum_total_term_freq, bool has_positions, bool has_norms, SniiScoringSegmentAccumulator* segment_accumulator); @@ -93,9 +93,15 @@ class CollectionStatistics { uint64_t get_term_doc_freq_by_col(const std::wstring& lucene_col_name, const std::wstring& term); uint64_t get_total_term_cnt_by_col(const std::wstring& lucene_col_name); - uint64_t get_doc_num() const; - - uint64_t _total_num_docs = 0; + uint64_t get_doc_num(const std::wstring& lucene_col_name) const; + + // Per field, the document count BM25 uses as N in idf and as the avgdl denominator. + // CLucene segments add their document count (the largest maxDoc among the segment's + // scored fields) to every field, as before per-field counts existed. SNII segments add + // the field's indexed (non-NULL) document count, which is what Lucene's per-field + // docCount means; a mostly NULL field would otherwise get a far too small avgdl and + // flattened idf. A field may count 0 documents. + std::unordered_map _total_num_docs; std::unordered_map _total_num_tokens; std::unordered_map> _term_doc_freqs; @@ -121,8 +127,9 @@ struct SniiScoringSegmentStats { // SNII scoring requires positions (which provide term frequencies) and norms. The current writer // emits norms for every analyzed index with positions. Older segments without norms return -// NOT_SUPPORTED until an index rebuild or compaction supplies them. -Result resolve_snii_scoring_segment(uint64_t index_doc_count, +// NOT_SUPPORTED until an index rebuild or compaction supplies them. The field's document count is +// its indexed (non-NULL) document count. +Result resolve_snii_scoring_segment(uint64_t indexed_doc_count, uint64_t sum_total_term_freq, bool has_positions, bool has_norms); diff --git a/be/src/storage/index/snii/bkd/bkd_format.h b/be/src/storage/index/snii/bkd/bkd_format.h index 63f6861c9ed11a..40ef8e35b1bbd5 100644 --- a/be/src/storage/index/snii/bkd/bkd_format.h +++ b/be/src/storage/index/snii/bkd/bkd_format.h @@ -62,7 +62,7 @@ inline constexpr uint32_t kSupportedVersion = 1; // no shared SectionType enum for blob logical indexes, so -- exactly as // format::kNullBitmapSectionType (0x20) does -- this is a documented literal // picked outside the ranges already taken by the inverted-index sections -// (format::SectionType, currently 1..14) and the null-bitmap POD (0x20). +// (format::SectionType, currently 1..15) and the null-bitmap POD (0x20). // Framing the payload is what gives bkd_index its checksum; no section here // hand-rolls a crc. inline constexpr uint8_t kBkdIndexSectionType = 0x30; diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.cpp b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp index efa021d2f1e843..e5028376bf7486 100644 --- a/be/src/storage/index/snii/compaction/snii_index_compaction.cpp +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.cpp @@ -390,6 +390,40 @@ Status SniiPlainT2MergePlan::write_current_term( return Status::OK(); } +Status SniiPlainT2MergePlan::encode_destination_norms( + size_t destination_segment, std::span null_docids, + // NOLINTNEXTLINE(readability-non-const-parameter): the vector is appended to below + std::vector* null_docids_with_norms, + writer::MemoryReporter::Reservation* null_docids_with_norms_reservation) { + // The accumulators hold each document's summed term frequencies, saturated at 255. + // Keep only the documents that carry a norm, exactly like the column writer: every + // non-NULL document, and a NULL document that still occurs in a posting (a nullable + // ARRAY row can keep tokens under its NULL flag). encode_norm maps 0 to 1, matching + // the writer's encode_norm(len) = clamp(len, 1, 255). + std::vector& norms = destination_encoded_norms_[destination_segment]; + DORIS_CHECK_EQ(norms.size(), destination_segment_num_rows_[destination_segment]); + DORIS_CHECK(null_docids_with_norms->empty()); + size_t kept = 0; + size_t next_null = 0; + for (size_t docid = 0; docid < norms.size(); ++docid) { + const uint8_t length = norms[docid]; + if (next_null < null_docids.size() && null_docids[next_null] == docid) { + ++next_null; + if (length == 0) { + continue; + } + RETURN_IF_ERROR(reserve_tracked_vector(null_docids_with_norms, 1, + null_docids_with_norms_reservation)); + null_docids_with_norms->push_back(static_cast(docid)); + } + norms[kept++] = query::encode_norm(length); + } + DORIS_CHECK_EQ(next_null, null_docids.size()); + // Shrinking keeps the capacity, so the transferred reservation still covers it. + norms.resize(kept); + return Status::OK(); +} + Status SniiPlainT2MergePlan::merge_terms( std::span sessions) { std::vector> term_cursors; @@ -412,15 +446,21 @@ Status SniiPlainT2MergePlan::merge_terms( } if (eligibility_.destination_writes_norms) { - // Accumulated raw lengths saturate at 255. encode_norm maps 0 to 1, matching the - // writer's encode_norm(len) = clamp(len, 1, 255). for (size_t destination_ordinal = 0; destination_ordinal < sessions.size(); ++destination_ordinal) { - for (uint8_t& value : destination_encoded_norms_[destination_ordinal]) { - value = query::encode_norm(value); - } + // The reservation precedes the vector so the vector is freed first. + writer::MemoryReporter::Reservation with_norms_reservation = + memory_reporter_ == nullptr ? writer::MemoryReporter::Reservation() + : memory_reporter_->make_reservation(); + std::vector null_docids_with_norms; + RETURN_IF_ERROR(encode_destination_norms( + destination_ordinal, sessions[destination_ordinal]->null_docids(), + &null_docids_with_norms, + memory_reporter_ == nullptr ? nullptr : &with_norms_reservation)); RETURN_IF_ERROR(sessions[destination_ordinal]->set_encoded_norms( - take_destination_encoded_norms(destination_ordinal))); + take_destination_encoded_norms(destination_ordinal), + writer::TrackedNullDocids(std::move(with_norms_reservation), + std::move(null_docids_with_norms)))); } } for (writer::SniiStreamedIndexSession* session : sessions) { diff --git a/be/src/storage/index/snii/compaction/snii_index_compaction.h b/be/src/storage/index/snii/compaction/snii_index_compaction.h index d70bd018cc66c7..76555acaaf893d 100644 --- a/be/src/storage/index/snii/compaction/snii_index_compaction.h +++ b/be/src/storage/index/snii/compaction/snii_index_compaction.h @@ -116,6 +116,13 @@ class SniiPlainT2MergePlan { Status write_current_term(CurrentTerm current, std::span sessions); Status merge_terms(std::span sessions); + // Turns the destination's accumulated lengths into the encoded norms of the documents + // that carry one and appends the NULL docids among them. The reservation (null without a + // memory reporter) covers the appended vector. + Status encode_destination_norms( + size_t destination_segment, std::span null_docids, + std::vector* null_docids_with_norms, + writer::MemoryReporter::Reservation* null_docids_with_norms_reservation); Status poison(Status status); std::vector source_indexes_; diff --git a/be/src/storage/index/snii/format/format_constants.h b/be/src/storage/index/snii/format/format_constants.h index 3073a18930e009..566cb2afc30897 100644 --- a/be/src/storage/index/snii/format/format_constants.h +++ b/be/src/storage/index/snii/format/format_constants.h @@ -17,6 +17,7 @@ #pragma once +#include #include // SNII container and per-section on-disk contract constants. @@ -61,6 +62,9 @@ enum class SectionType : uint8_t { // Core metadata so a corrupt section reference cannot reinterpret valid // collection statistics as document norms. kNormsPod = 14, + // BM25 norms stored only for the documents that carry one (see norms_pod.h). Readers that + // predate it reject the section instead of misreading it as kNormsPod. + kNormsSparse = 15, }; // ---- Logical index postings storage content configuration (fixed per logical diff --git a/be/src/storage/index/snii/format/norms_pod.cpp b/be/src/storage/index/snii/format/norms_pod.cpp index b3da050c17726f..35406f953ba4ec 100644 --- a/be/src/storage/index/snii/format/norms_pod.cpp +++ b/be/src/storage/index/snii/format/norms_pod.cpp @@ -17,8 +17,13 @@ #include "storage/index/snii/format/norms_pod.h" +#include +#include +#include #include +#include +#include "common/check.h" #include "storage/index/snii/common/slice.h" #include "storage/index/snii/encoding/byte_source.h" #include "storage/index/snii/encoding/section_framer.h" @@ -27,6 +32,436 @@ namespace doris::snii::format { +namespace { + +constexpr uint64_t kBlockSpan = uint64_t {1} << 16; +constexpr size_t kBlockHeaderBytes = 12; +constexpr size_t kBitsetWords = 1024; +constexpr size_t kBitsetRankEntries = 128; +constexpr size_t kWordsPerRankEntry = kBitsetWords / kBitsetRankEntries; +constexpr size_t kBitsetBlockBytes = + kBitsetWords * sizeof(uint64_t) + kBitsetRankEntries * sizeof(uint16_t); +constexpr size_t kRunBytes = 3 * sizeof(uint16_t); + +enum class BlockKind : uint8_t { kAll = 0, kArray = 1, kBitset = 2, kRuns = 3 }; + +Status corrupted(std::string_view reason) { + return Status::Error("norms: {}", reason); +} + +Status invalid_input(std::string_view reason) { + return Status::Error("norms: {}", reason); +} + +uint16_t load16(const uint8_t* p) { + return static_cast(p[0] | (static_cast(p[1]) << 8)); +} + +uint32_t load32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +uint64_t load64(const uint8_t* p) { + return static_cast(load32(p)) | (static_cast(load32(p + 4)) << 32); +} + +size_t framed_section_bytes(size_t payload_bytes) { + return 1 + varint_len(payload_bytes) + payload_bytes + sizeof(uint32_t); +} + +// Ascending docids of the documents without a norm: null_docids minus null_docids_with_norms. +// The input is validated by plan_norms_section, so null_docids_with_norms is a subset. +class NormlessDocids { +public: + NormlessDocids(std::span null_docids, + std::span null_docids_with_norms) + : null_docids_(null_docids), null_docids_with_norms_(null_docids_with_norms) { + skip_null_docids_with_norms(); + } + + bool at_end() const { return null_index_ == null_docids_.size(); } + uint32_t value() const { return null_docids_[null_index_]; } + void advance() { + ++null_index_; + skip_null_docids_with_norms(); + } + +private: + void skip_null_docids_with_norms() { + while (null_index_ < null_docids_.size() && + with_norms_index_ < null_docids_with_norms_.size() && + null_docids_[null_index_] == null_docids_with_norms_[with_norms_index_]) { + ++null_index_; + ++with_norms_index_; + } + } + + std::span null_docids_; + std::span null_docids_with_norms_; + size_t null_index_ = 0; + size_t with_norms_index_ = 0; +}; + +// One SPARSE block holding at least one present docid. +struct PresentBlock { + uint32_t key; + uint32_t span; + uint32_t cardinality; + uint32_t runs; + BlockKind kind; + size_t payload_bytes; + // Positioned at the first docid without a norm at or after the block start. + NormlessDocids normless; +}; + +// Visits the non-empty blocks in key order. Work is O(doc_count / 65536 + normless docids). +template +void for_each_present_block(uint32_t doc_count, NormlessDocids normless, Fn&& fn) { + for (uint64_t base = 0; base < doc_count; base += kBlockSpan) { + const auto span = static_cast(std::min(kBlockSpan, doc_count - base)); + const NormlessDocids block_normless = normless; + uint32_t absent = 0; + uint32_t runs = 0; + uint32_t next_low = 0; + while (!normless.at_end() && normless.value() < base + span) { + const auto low = static_cast(normless.value() - base); + if (low > next_low) { + ++runs; + } + next_low = low + 1; + ++absent; + normless.advance(); + } + if (next_low < span) { + ++runs; + } + const uint32_t cardinality = span - absent; + if (cardinality == 0) { + continue; + } + BlockKind kind = BlockKind::kAll; + size_t payload_bytes = 0; + if (absent != 0) { + const size_t array_bytes = size_t {cardinality} * sizeof(uint16_t); + const size_t run_bytes = sizeof(uint16_t) + size_t {runs} * kRunBytes; + if (array_bytes <= std::min(run_bytes, kBitsetBlockBytes)) { + kind = BlockKind::kArray; + payload_bytes = array_bytes; + } else if (run_bytes <= kBitsetBlockBytes) { + kind = BlockKind::kRuns; + payload_bytes = run_bytes; + } else { + kind = BlockKind::kBitset; + payload_bytes = kBitsetBlockBytes; + } + } + fn(PresentBlock {.key = static_cast(base >> 16), + .span = span, + .cardinality = cardinality, + .runs = runs, + .kind = kind, + .payload_bytes = payload_bytes, + .normless = block_normless}); + } +} + +// Calls fn(first_low, last_low) for every maximal run of present lows of the block. +template +void for_each_present_run(const PresentBlock& block, Fn&& fn) { + NormlessDocids normless = block.normless; + const uint64_t base = uint64_t {block.key} << 16; + uint32_t next_low = 0; + while (!normless.at_end() && normless.value() < base + block.span) { + const auto low = static_cast(normless.value() - base); + if (low > next_low) { + fn(next_low, low - 1); + } + next_low = low + 1; + normless.advance(); + } + if (next_low < block.span) { + fn(next_low, block.span - 1); + } +} + +void write_block_payload(const PresentBlock& block, ByteSink* out) { + switch (block.kind) { + case BlockKind::kAll: + return; + case BlockKind::kArray: + for_each_present_run(block, [out](uint32_t first, uint32_t last) { + for (uint32_t low = first; low <= last; ++low) { + out->put_fixed16(static_cast(low)); + } + }); + return; + case BlockKind::kRuns: { + out->put_fixed16(static_cast(block.runs)); + uint32_t rank_before = 0; + for_each_present_run(block, [out, &rank_before](uint32_t first, uint32_t last) { + out->put_fixed16(static_cast(first)); + out->put_fixed16(static_cast(last)); + out->put_fixed16(static_cast(rank_before)); + rank_before += last - first + 1; + }); + DORIS_CHECK_EQ(rank_before, block.cardinality); + return; + } + case BlockKind::kBitset: { + std::array words {}; + for_each_present_run(block, [&words](uint32_t first, uint32_t last) { + for (uint32_t low = first; low <= last;) { + if ((low & 63) == 0 && last - low >= 63) { + words[low >> 6] = ~uint64_t {0}; + low += 64; + } else { + words[low >> 6] |= uint64_t {1} << (low & 63); + ++low; + } + } + }); + for (uint64_t word : words) { + out->put_fixed64(word); + } + uint32_t rank = 0; + for (size_t word = 0; word < kBitsetWords; ++word) { + if (word % kWordsPerRankEntry == 0) { + out->put_fixed16(static_cast(rank)); + } + rank += std::popcount(words[word]); + } + DORIS_CHECK_EQ(rank, block.cardinality); + return; + } + } + DORIS_CHECK(false); +} + +Status validate_array_block(const uint8_t* payload, uint64_t available, uint32_t span, + uint32_t cardinality, size_t* payload_bytes) { + *payload_bytes = size_t {cardinality} * sizeof(uint16_t); + if (*payload_bytes > available) { + return corrupted("ARRAY block past block data"); + } + uint32_t previous = 0; + for (uint32_t i = 0; i < cardinality; ++i) { + const uint32_t low = load16(payload + i * sizeof(uint16_t)); + if ((i != 0 && low <= previous) || low >= span) { + return corrupted("ARRAY block values not ascending inside the span"); + } + previous = low; + } + return Status::OK(); +} + +Status validate_bitset_block(const uint8_t* payload, uint64_t available, uint32_t span, + uint32_t cardinality, size_t* payload_bytes) { + *payload_bytes = kBitsetBlockBytes; + if (*payload_bytes > available) { + return corrupted("BITSET block past block data"); + } + const uint8_t* rank_table = payload + kBitsetWords * sizeof(uint64_t); + uint32_t rank = 0; + for (size_t word_index = 0; word_index < kBitsetWords; ++word_index) { + if (word_index % kWordsPerRankEntry == 0 && + load16(rank_table + word_index / kWordsPerRankEntry * sizeof(uint16_t)) != rank) { + return corrupted("BITSET block rank table mismatch"); + } + const uint64_t word = load64(payload + word_index * sizeof(uint64_t)); + const uint64_t word_base = word_index * 64; + const bool bits_past_span = + word_base >= span ? word != 0 + : span - word_base < 64 && (word >> (span - word_base)) != 0; + if (bits_past_span) { + return corrupted("BITSET block has bits past its span"); + } + rank += std::popcount(word); + } + if (rank != cardinality) { + return corrupted("BITSET block cardinality mismatch"); + } + return Status::OK(); +} + +Status validate_runs_block(const uint8_t* payload, uint64_t available, uint32_t span, + uint32_t cardinality, size_t* payload_bytes) { + if (available < sizeof(uint16_t)) { + return corrupted("RUNS block past block data"); + } + const uint32_t run_count = load16(payload); + *payload_bytes = sizeof(uint16_t) + size_t {run_count} * kRunBytes; + if (run_count == 0 || *payload_bytes > available) { + return corrupted("RUNS block run count out of range"); + } + uint32_t rank_before = 0; + uint32_t previous_last = 0; + for (uint32_t run = 0; run < run_count; ++run) { + const uint8_t* entry = payload + sizeof(uint16_t) + run * kRunBytes; + const uint32_t first = load16(entry); + const uint32_t last = load16(entry + sizeof(uint16_t)); + if (first > last || last >= span || (run != 0 && first <= previous_last + 1)) { + return corrupted("RUNS block runs not ascending and separated inside the span"); + } + if (load16(entry + 2 * sizeof(uint16_t)) != rank_before) { + return corrupted("RUNS block rank mismatch"); + } + rank_before += last - first + 1; + previous_last = last; + } + if (rank_before != cardinality) { + return corrupted("RUNS block cardinality mismatch"); + } + return Status::OK(); +} + +// Validates one block payload and returns its length. +Status validate_block(BlockKind kind, Slice block_data, uint64_t offset, uint32_t span, + uint32_t cardinality, size_t* payload_bytes) { + if (offset > block_data.size()) { + return corrupted("block payload offset past block data"); + } + const uint64_t available = block_data.size() - offset; + const uint8_t* payload = block_data.data() + offset; + switch (kind) { + case BlockKind::kAll: + if (cardinality != span) { + return corrupted("ALL block cardinality differs from its span"); + } + *payload_bytes = 0; + return Status::OK(); + case BlockKind::kArray: + return validate_array_block(payload, available, span, cardinality, payload_bytes); + case BlockKind::kBitset: + return validate_bitset_block(payload, available, span, cardinality, payload_bytes); + case BlockKind::kRuns: + return validate_runs_block(payload, available, span, cardinality, payload_bytes); + } + return corrupted("unknown block kind"); +} + +// Validates the block headers and payloads of a sparse section against its present count and +// block data length. +Status validate_sparse_blocks(uint64_t doc_count, uint64_t present_count, Slice headers, + uint64_t block_count, Slice block_data) { + uint64_t expected_rank = 0; + uint64_t expected_offset = 0; + for (uint64_t block = 0; block < block_count; ++block) { + const uint8_t* header = headers.data() + block * kBlockHeaderBytes; + const uint32_t key = load16(header); + const uint8_t kind = header[2]; + if (kind > static_cast(BlockKind::kRuns) || header[3] != 0) { + return corrupted("invalid block header"); + } + if (block != 0 && key <= load16(header - kBlockHeaderBytes)) { + return corrupted("block keys not ascending"); + } + const uint64_t base = uint64_t {key} << 16; + if (base >= doc_count) { + return corrupted("block outside the document domain"); + } + if (load32(header + 4) != expected_rank || load32(header + 8) != expected_offset) { + return corrupted("block rank or payload offset mismatch"); + } + const uint64_t next_rank = + block + 1 < block_count ? load32(header + kBlockHeaderBytes + 4) : present_count; + const auto span = static_cast(std::min(kBlockSpan, doc_count - base)); + if (next_rank <= expected_rank || next_rank - expected_rank > span) { + return corrupted("block cardinality out of range"); + } + size_t block_payload_bytes = 0; + RETURN_IF_ERROR(validate_block(static_cast(kind), block_data, expected_offset, + span, static_cast(next_rank - expected_rank), + &block_payload_bytes)); + expected_rank = next_rank; + expected_offset += block_payload_bytes; + } + if (expected_rank != present_count) { + return corrupted("block cardinalities differ from present count"); + } + if (expected_offset != block_data.size()) { + return corrupted("block payloads differ from block data length"); + } + return Status::OK(); +} + +// Index of low among the present lows of a block, or false when low is absent. +bool array_block_rank(const uint8_t* payload, uint32_t cardinality, uint32_t low, uint32_t* rank) { + uint32_t first = 0; + uint32_t count = cardinality; + while (count > 0) { + const uint32_t half = count / 2; + if (load16(payload + (first + half) * sizeof(uint16_t)) < low) { + first += half + 1; + count -= half + 1; + } else { + count = half; + } + } + if (first == cardinality || load16(payload + first * sizeof(uint16_t)) != low) { + return false; + } + *rank = first; + return true; +} + +bool bitset_block_rank(const uint8_t* payload, uint32_t low, uint32_t* rank) { + const uint32_t word_index = low >> 6; + const uint64_t word = load64(payload + word_index * sizeof(uint64_t)); + if (((word >> (low & 63)) & 1) == 0) { + return false; + } + const uint32_t group = low >> 9; + uint32_t in_block = + load16(payload + kBitsetWords * sizeof(uint64_t) + group * sizeof(uint16_t)); + for (uint32_t w = group * kWordsPerRankEntry; w < word_index; ++w) { + in_block += std::popcount(load64(payload + w * sizeof(uint64_t))); + } + in_block += std::popcount(word & ((uint64_t {1} << (low & 63)) - 1)); + *rank = in_block; + return true; +} + +bool runs_block_rank(const uint8_t* payload, uint32_t low, uint32_t* rank) { + // The last run whose first low is <= low. + uint32_t after = 0; + uint32_t count = load16(payload); + while (count > 0) { + const uint32_t half = count / 2; + if (load16(payload + sizeof(uint16_t) + (after + half) * kRunBytes) <= low) { + after += half + 1; + count -= half + 1; + } else { + count = half; + } + } + if (after == 0) { + return false; + } + const uint8_t* entry = payload + sizeof(uint16_t) + (after - 1) * kRunBytes; + const uint32_t run_first = load16(entry); + if (low > load16(entry + sizeof(uint16_t))) { + return false; + } + *rank = load16(entry + 2 * sizeof(uint16_t)) + (low - run_first); + return true; +} + +Status get_canonical_doc_count(ByteSource* payload, uint64_t* doc_count) { + RETURN_IF_ERROR(payload->get_varint64(doc_count)); + if (payload->position() != varint_len(*doc_count)) { + return Status::Error( + "norms POD non-canonical doc_count"); + } + if (*doc_count > std::numeric_limits::max()) { + return Status::Error( + "norms POD doc_count overflows uint32"); + } + return Status::OK(); +} + +} // namespace + void NormsPodWriter::finish(ByteSink* sink) const { finish(norms_, sink); } @@ -39,16 +474,145 @@ void NormsPodWriter::finish(std::span norms, ByteSink* sink) { payload.put_varint64(norms.size()); payload.put_bytes(Slice(norms.data(), norms.size())); // Delegate outer framing to SectionFramer to append type+len+crc32c, avoiding manual checksum assembly. - sink->reserve(1 + varint_len(payload_size) + payload_size + sizeof(uint32_t)); + sink->reserve(framed_section_bytes(payload_size)); SectionFramer::write(*sink, static_cast(SectionType::kNormsPod), payload.view()); } +uint64_t dense_norms_section_bytes(uint64_t doc_count) { + return framed_section_bytes(varint_len(doc_count) + doc_count); +} + +Status plan_norms_section(const NormsSectionInput& in, bool force_dense, NormsSectionPlan* out) { + for (size_t i = 0; i < in.null_docids.size(); ++i) { + if (in.null_docids[i] >= in.doc_count || + (i != 0 && in.null_docids[i] <= in.null_docids[i - 1])) { + return invalid_input("null docids must be ascending and inside the document domain"); + } + } + size_t null_index = 0; + for (size_t i = 0; i < in.null_docids_with_norms.size(); ++i) { + const uint32_t docid = in.null_docids_with_norms[i]; + while (null_index < in.null_docids.size() && in.null_docids[null_index] < docid) { + ++null_index; + } + if (null_index == in.null_docids.size() || in.null_docids[null_index] != docid) { + return invalid_input( + "NULL docids with norms must be an ascending subset of null docids"); + } + ++null_index; + } + const uint64_t present_count = + uint64_t {in.doc_count} - in.null_docids.size() + in.null_docids_with_norms.size(); + if (in.norms.size() != present_count) { + return invalid_input("norm count differs from the documents that carry a norm"); + } + + NormsSectionPlan plan; + plan.payload_bytes = varint_len(in.doc_count) + in.doc_count; + plan.framed_bytes = dense_norms_section_bytes(in.doc_count); + if (force_dense || present_count == in.doc_count) { + *out = plan; + return Status::OK(); + } + + const uint8_t constant_norm = in.norms.empty() ? 0 : in.norms.front(); + const bool all_equal = + std::all_of(in.norms.begin(), in.norms.end(), + [constant_norm](uint8_t norm) { return norm == constant_norm; }); + size_t block_count = 0; + size_t block_data_bytes = 0; + for_each_present_block(in.doc_count, NormlessDocids(in.null_docids, in.null_docids_with_norms), + [&](const PresentBlock& block) { + ++block_count; + block_data_bytes += block.payload_bytes; + }); + const size_t norm_bytes = all_equal ? 1 : present_count; + const size_t payload_bytes = varint_len(in.doc_count) + varint_len(present_count) + 1 + + varint_len(block_count) + block_count * kBlockHeaderBytes + + varint_len(block_data_bytes) + block_data_bytes + norm_bytes; + const size_t framed_bytes = framed_section_bytes(payload_bytes); + if (framed_bytes < plan.framed_bytes) { + plan = NormsSectionPlan {.layout = NormsLayout::kSparse, + .bytes_per_norm = static_cast(all_equal ? 0 : 1), + .constant_norm = all_equal ? constant_norm : uint8_t {0}, + .payload_bytes = payload_bytes, + .framed_bytes = framed_bytes}; + } + *out = plan; + return Status::OK(); +} + +void write_norms_section(const NormsSectionInput& in, const NormsSectionPlan& plan, + ByteSink* sink) { + ByteSink payload; + payload.reserve(plan.payload_bytes); + payload.put_varint64(in.doc_count); + const NormlessDocids normless_begin(in.null_docids, in.null_docids_with_norms); + auto section_type = static_cast(SectionType::kNormsPod); + if (plan.layout == NormsLayout::kDense) { + // Documents without a norm keep kEmptyDocumentNorm, the byte every earlier writer + // stored for them, so the section is identical to theirs. + size_t norm_index = 0; + uint32_t next_docid = 0; + for (NormlessDocids normless = normless_begin; !normless.at_end(); normless.advance()) { + const uint32_t docid = normless.value(); + payload.put_bytes(Slice(in.norms.data() + norm_index, docid - next_docid)); + norm_index += docid - next_docid; + payload.put_u8(kEmptyDocumentNorm); + next_docid = docid + 1; + } + DORIS_CHECK_EQ(in.norms.size() - norm_index, in.doc_count - next_docid); + payload.put_bytes(Slice(in.norms.data() + norm_index, in.norms.size() - norm_index)); + } else { + section_type = static_cast(SectionType::kNormsSparse); + payload.put_varint64(in.norms.size()); + payload.put_u8(plan.bytes_per_norm); + size_t block_count = 0; + size_t block_data_bytes = 0; + for_each_present_block(in.doc_count, normless_begin, [&](const PresentBlock& block) { + ++block_count; + block_data_bytes += block.payload_bytes; + }); + payload.put_varint64(block_count); + uint32_t rank_base = 0; + size_t payload_offset = 0; + for_each_present_block(in.doc_count, normless_begin, [&](const PresentBlock& block) { + payload.put_fixed16(static_cast(block.key)); + payload.put_u8(static_cast(block.kind)); + payload.put_u8(0); + payload.put_fixed32(rank_base); + payload.put_fixed32(static_cast(payload_offset)); + rank_base += block.cardinality; + payload_offset += block.payload_bytes; + }); + DORIS_CHECK_EQ(rank_base, in.norms.size()); + payload.put_varint64(block_data_bytes); + const size_t block_data_begin = payload.size(); + for_each_present_block(in.doc_count, normless_begin, [&](const PresentBlock& block) { + write_block_payload(block, &payload); + }); + DORIS_CHECK_EQ(payload.size() - block_data_begin, block_data_bytes); + if (plan.bytes_per_norm == 0) { + payload.put_u8(plan.constant_norm); + } else { + payload.put_bytes(Slice(in.norms.data(), in.norms.size())); + } + } + DORIS_CHECK_EQ(payload.size(), plan.payload_bytes); + + const size_t start = sink->size(); + sink->reserve(plan.framed_bytes); + SectionFramer::write(*sink, section_type, payload.view()); + DORIS_CHECK_EQ(sink->size() - start, plan.framed_bytes); +} + Status NormsPodReader::open(Slice framed, NormsPodReader* out) { // framer handles CRC verify, truncation detection, and payload slicing. ByteSource src(framed); FramedSection sec; RETURN_IF_ERROR(SectionFramer::read(src, &sec)); - if (sec.type != static_cast(SectionType::kNormsPod)) { + if (sec.type != static_cast(SectionType::kNormsPod) && + sec.type != static_cast(SectionType::kNormsSparse)) { return Status::Error( "norms POD section type mismatch"); } @@ -56,19 +620,14 @@ Status NormsPodReader::open(Slice framed, NormsPodReader* out) { return Status::Error( "norms POD trailing framed bytes"); } + if (sec.type == static_cast(SectionType::kNormsSparse)) { + return open_sparse(sec.payload, out); + } // Parse inner payload: [varint64 doc_count][bytes]. ByteSource payload(sec.payload); uint64_t doc_count = 0; - RETURN_IF_ERROR(payload.get_varint64(&doc_count)); - if (payload.position() != varint_len(doc_count)) { - return Status::Error( - "norms POD non-canonical doc_count"); - } - if (doc_count > std::numeric_limits::max()) { - return Status::Error( - "norms POD doc_count overflows uint32"); - } + RETURN_IF_ERROR(get_canonical_doc_count(&payload, &doc_count)); // doc_count must exactly equal the remaining byte count (1 byte per doc). if (payload.remaining() != doc_count) { return Status::Error( @@ -77,8 +636,132 @@ Status NormsPodReader::open(Slice framed, NormsPodReader* out) { Slice bytes; RETURN_IF_ERROR(payload.get_bytes(static_cast(doc_count), &bytes)); - out->doc_count_ = static_cast(doc_count); - out->norms_ = bytes.data(); + NormsPodReader reader; + reader.layout_ = NormsLayout::kDense; + reader.doc_count_ = static_cast(doc_count); + reader.present_count_ = reader.doc_count_; + reader.norms_ = bytes.data(); + *out = reader; + return Status::OK(); +} + +Status NormsPodReader::open_sparse(Slice payload_bytes, NormsPodReader* out) { + ByteSource payload(payload_bytes); + uint64_t doc_count = 0; + RETURN_IF_ERROR(get_canonical_doc_count(&payload, &doc_count)); + uint64_t present_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&present_count)); + if (present_count > doc_count) { + return corrupted("present count exceeds doc count"); + } + uint8_t bytes_per_norm = 0; + RETURN_IF_ERROR(payload.get_u8(&bytes_per_norm)); + if (bytes_per_norm > 1) { + return corrupted("bytes_per_norm out of range"); + } + uint64_t block_count = 0; + RETURN_IF_ERROR(payload.get_varint64(&block_count)); + if (block_count > payload.remaining() / kBlockHeaderBytes) { + return corrupted("block headers past section end"); + } + Slice headers; + RETURN_IF_ERROR( + payload.get_bytes(static_cast(block_count * kBlockHeaderBytes), &headers)); + uint64_t block_data_bytes = 0; + RETURN_IF_ERROR(payload.get_varint64(&block_data_bytes)); + Slice block_data; + RETURN_IF_ERROR(payload.get_bytes(static_cast(block_data_bytes), &block_data)); + uint8_t constant_norm = 0; + Slice norms; + if (bytes_per_norm == 0) { + RETURN_IF_ERROR(payload.get_u8(&constant_norm)); + } else { + RETURN_IF_ERROR(payload.get_bytes(static_cast(present_count), &norms)); + } + if (!payload.eof()) { + return corrupted("trailing sparse payload bytes"); + } + + RETURN_IF_ERROR( + validate_sparse_blocks(doc_count, present_count, headers, block_count, block_data)); + + NormsPodReader reader; + reader.layout_ = NormsLayout::kSparse; + reader.doc_count_ = static_cast(doc_count); + reader.present_count_ = static_cast(present_count); + reader.norms_ = norms.data(); + reader.bytes_per_norm_ = bytes_per_norm; + reader.constant_norm_ = constant_norm; + reader.block_count_ = static_cast(block_count); + reader.block_headers_ = headers.data(); + reader.block_data_ = block_data.data(); + *out = reader; + return Status::OK(); +} + +bool NormsPodReader::sparse_rank(uint32_t docid, uint32_t* rank) const { + const uint32_t key = docid >> 16; + const uint32_t low = docid & 0xFFFFU; + uint32_t block = 0; + uint32_t count = block_count_; + while (count > 0) { + const uint32_t half = count / 2; + if (load16(block_headers_ + (block + half) * kBlockHeaderBytes) < key) { + block += half + 1; + count -= half + 1; + } else { + count = half; + } + } + if (block == block_count_) { + return false; + } + const uint8_t* header = block_headers_ + block * kBlockHeaderBytes; + if (load16(header) != key) { + return false; + } + const uint32_t rank_base = load32(header + 4); + const uint8_t* payload = block_data_ + load32(header + 8); + uint32_t in_block = 0; + bool present = false; + switch (static_cast(header[2])) { + case BlockKind::kAll: + in_block = low; + present = true; + break; + case BlockKind::kArray: { + const uint32_t next_rank = + block + 1 < block_count_ ? load32(header + kBlockHeaderBytes + 4) : present_count_; + present = array_block_rank(payload, next_rank - rank_base, low, &in_block); + break; + } + case BlockKind::kBitset: + present = bitset_block_rank(payload, low, &in_block); + break; + case BlockKind::kRuns: + present = runs_block_rank(payload, low, &in_block); + break; + } + *rank = rank_base + in_block; + return present; +} + +uint8_t NormsPodReader::sparse_encoded_norm(uint32_t docid) const { + uint32_t rank = 0; + const bool has_norm = sparse_rank(docid, &rank); + // Per-posting hot path: the docid comes from a decoded posting, so it carries a norm. For a + // docid without one, rank stays inside the section (0 or the rank base of a stored block). + DCHECK(has_norm); + return bytes_per_norm_ == 0 ? constant_norm_ : norms_[rank]; +} + +Status NormsPodReader::try_sparse_encoded_norm(uint32_t docid, uint8_t* out) const { + uint32_t rank = 0; + if (!sparse_rank(docid, &rank)) { + return Status::Error( + "norms: docid {} carries no norm", docid); + } + *out = bytes_per_norm_ == 0 ? constant_norm_ : norms_[rank]; return Status::OK(); } diff --git a/be/src/storage/index/snii/format/norms_pod.h b/be/src/storage/index/snii/format/norms_pod.h index 11791240615a65..a7d3e2f060575a 100644 --- a/be/src/storage/index/snii/format/norms_pod.h +++ b/be/src/storage/index/snii/format/norms_pod.h @@ -29,13 +29,60 @@ namespace doris::snii::format { -// norms POD: per logical index / field stores 1-byte encoded doc length per doc, -// used by BM25 length normalization (SniiStatsProvider::encoded_norm) for per-docid lookup. +// Norms of one logical index: the 1-byte encoded document length BM25 length normalization +// reads per docid (SniiStatsProvider::encoded_norm). A document carries a norm when it is not +// NULL, or when it is NULL but still produced tokens (a nullable ARRAY row may keep its nested +// payload under the NULL flag; its tokens sit in the postings exactly as for a non-NULL row). +// Postings only reference documents that carry a norm. // -// On-disk layout (the whole section is framed by SectionFramer, which adds a type+len+crc32c envelope): -// framer payload = [varint64 doc_count][bytes encoded_norm[doc_count]] +// Two section layouts exist. Both are framed by SectionFramer: // framer envelope = [u8 type][varint64 payload_len][payload][fixed32 crc32c] -// The encoding of encoded_norm (length -> 1B) is out of scope for this module; here we only handle raw byte storage and retrieval. +// +// DENSE (type SectionType::kNormsPod = 14), one byte per document: +// payload = [varint64 doc_count][doc_count bytes: encoded_norm[docid]] +// A document without a norm stores kEmptyDocumentNorm. This is the only layout readers before +// the sparse layout understand, and the writer keeps it for every logical index without NULL +// rows, so those sections stay byte-identical to the earlier format. +// +// SPARSE (type SectionType::kNormsSparse = 15), bytes only for documents with a norm: +// payload = [varint64 doc_count] +// [varint64 present_count] documents that carry a norm +// [u8 bytes_per_norm] 0 (all norms equal) or 1 +// [varint64 block_count] +// [block_count x 12-byte block header] +// [varint64 block_data_len] +// [block_data_len bytes: block payloads, in header order, without gaps] +// bytes_per_norm == 0: [u8 constant norm] (written as 0 when present_count == 0) +// bytes_per_norm == 1: [present_count bytes: norms in ascending docid order] +// The present docids are split into blocks of 65536 docids (block key = docid >> 16); only +// blocks with at least one present docid are stored, in ascending key order. A block header is +// [fixed16 key][u8 kind][u8 reserved = 0][fixed32 rank_base][fixed32 payload_offset] +// rank_base is the number of present docids in the preceding blocks (the index of the block's +// first norm), payload_offset the offset of the block payload inside the block data. A block's +// cardinality is the next header's rank_base (present_count for the last block) minus its own. +// With span = min(65536, doc_count - (key << 16)) and low = docid & 0xFFFF, the kinds are +// 0 ALL: every docid of the block is present; no payload; cardinality == span. +// 1 ARRAY: [cardinality x fixed16 low], strictly ascending. +// 2 BITSET: [1024 x fixed64 words][128 x fixed16 rank]; bit (low & 63) of word (low >> 6) is +// set for a present low, no bit at or past span is set, and rank[g] is the number +// of set bits in words [0, 8 * g). +// 3 RUNS: [fixed16 run_count][run_count x (fixed16 first_low, fixed16 last_low, +// fixed16 rank_before)], runs ascending and separated by at least one absent low; +// rank_before is the number of present lows in the preceding runs. +// The writer picks, per block, ALL when nothing is absent and otherwise the smallest of ARRAY, +// RUNS and BITSET (ties in that order). A lookup costs a binary search over the block headers +// plus O(1) (ALL, BITSET) or a binary search inside the block (ARRAY, RUNS), and reads only the +// section bytes, so the reader needs no memory beyond the section itself. +// +// The writer picks the layout when the logical index is finished: DENSE when no document lacks a +// norm or the BE config enable_snii_sparse_norms is off, otherwise the smaller of SPARSE and DENSE +// (DENSE on a tie). Readers accept both layouts whatever the config says. + +// Norm stored by the dense layout for a document without a norm. Equals query::encode_norm(0), +// which is what every writer stored for such rows before the sparse layout existed. +inline constexpr uint8_t kEmptyDocumentNorm = 1; + +// Accumulates dense norms one document at a time (docid is the append order). class NormsPodWriter { public: // Appends the encoded_norm for the next docid (docid is implicit, assigned in append order starting from 0). @@ -44,46 +91,112 @@ class NormsPodWriter { // Number of docs accumulated so far (i.e., the next docid to be assigned). size_t count() const { return norms_.size(); } - // Writes [doc_count][bytes] framed by SectionFramer into sink (appends; does not clear sink). + // Writes the DENSE section [doc_count][bytes] framed by SectionFramer into sink (appends; does + // not clear sink). void finish(ByteSink* sink) const; - // Zero-copy source overload used by streamed compaction. + // Zero-copy source overload: one norm per document. static void finish(std::span norms, ByteSink* sink); private: std::vector norms_; }; -// Read-only view: on open, verifies the framer CRC and checks that doc_count/payload length are consistent, -// afterwards encoded_norm(docid) is O(1) direct indexing (zero-copy, borrows the underlying buffer). +// Norms of one logical index as handed to the section writer. +struct NormsSectionInput { + uint32_t doc_count = 0; + // Ascending NULL docids. + std::span null_docids; + // Ascending subset of null_docids whose documents still carry a norm. + std::span null_docids_with_norms; + // Encoded norms of the documents that carry one (every docid outside null_docids plus + // null_docids_with_norms), in ascending docid order. + std::span norms; +}; + +enum class NormsLayout : uint8_t { kDense, kSparse }; + +struct NormsSectionPlan { + NormsLayout layout = NormsLayout::kDense; + // SPARSE only. + uint8_t bytes_per_norm = 1; + uint8_t constant_norm = 0; + size_t payload_bytes = 0; + size_t framed_bytes = 0; +}; + +// Validates the input shape (sorted docids inside the document domain, null_docids_with_norms a +// subset of null_docids, one norm per document that carries one) and picks the layout described +// above. force_dense selects DENSE regardless of size (LogicalIndexWriter::finalize_build passes +// !config::enable_snii_sparse_norms). +Status plan_norms_section(const NormsSectionInput& in, bool force_dense, NormsSectionPlan* out); + +// Appends the planned section to sink: exactly plan.framed_bytes bytes. The input must be the +// one the plan was computed from. +void write_norms_section(const NormsSectionInput& in, const NormsSectionPlan& plan, ByteSink* sink); + +// Framed length of the DENSE section for doc_count documents. A valid norms section of either +// layout is never longer (the writer only chooses SPARSE when it is shorter). +uint64_t dense_norms_section_bytes(uint64_t doc_count); + +// Read-only view over a DENSE or SPARSE section. open() verifies the framer CRC and the complete +// layout; afterwards lookups only read the borrowed section bytes, so copies are cheap and the +// caller must keep the bytes alive. class NormsPodReader { public: NormsPodReader() = default; - // Parses the entire section (including the framer envelope). Returns Corruption on CRC mismatch, truncation, or length inconsistency. - // On success, *out borrows the memory pointed to by framer_payload; the caller must ensure its lifetime. + // Parses the entire section (including the framer envelope). Returns Corruption on CRC + // mismatch, truncation, an unknown section type, or any layout inconsistency. static Status open(Slice framed, NormsPodReader* out); uint32_t doc_count() const { return doc_count_; } + bool is_sparse() const { return layout_ == NormsLayout::kSparse; } + // Documents that carry a norm in a SPARSE section; doc_count() for a DENSE section, which + // stores a byte for every document. + uint32_t present_count() const { return present_count_; } - // Precondition (hard contract): docid < doc_count(). Semantics match std::vector::operator[]: - // the caller is responsible for guaranteeing this (docid comes from trusted postings decoded internally by SNII). Asserts in debug builds; - // no check in Release (NDEBUG). Use try_encoded_norm when the docid is untrusted and needs validation. + // Precondition (hard contract): docid < doc_count() and docid carries a norm (it comes from a + // posting decoded internally by SNII). Both layouts only assert it in debug builds; use + // try_encoded_norm when the docid is untrusted. uint8_t encoded_norm(uint32_t docid) const { assert(docid < doc_count_); - return norms_[docid]; + if (layout_ == NormsLayout::kDense) { + return norms_[docid]; + } + return sparse_encoded_norm(docid); } - // Checked access: returns InvalidArgument if docid is out of range; never reads out-of-range memory. + // Checked access: InvalidArgument for a docid out of range, Corruption for a docid without a + // norm in a SPARSE section; never reads out-of-range memory. Status try_encoded_norm(uint32_t docid, uint8_t* out) const { - if (docid >= doc_count_) + if (docid >= doc_count_) { return Status::Error("norms: docid out of range"); - *out = norms_[docid]; - return Status::OK(); + } + if (layout_ == NormsLayout::kDense) { + *out = norms_[docid]; + return Status::OK(); + } + return try_sparse_encoded_norm(docid, out); } private: - const uint8_t* norms_ = nullptr; + static Status open_sparse(Slice payload, NormsPodReader* out); + // Index of docid among the present docids; false when docid carries no norm. + bool sparse_rank(uint32_t docid, uint32_t* rank) const; + uint8_t sparse_encoded_norm(uint32_t docid) const; + Status try_sparse_encoded_norm(uint32_t docid, uint8_t* out) const; + + NormsLayout layout_ = NormsLayout::kDense; uint32_t doc_count_ = 0; + uint32_t present_count_ = 0; + // DENSE: doc_count_ bytes. SPARSE: present_count_ bytes when bytes_per_norm_ == 1. + const uint8_t* norms_ = nullptr; + // SPARSE only. + uint8_t bytes_per_norm_ = 0; + uint8_t constant_norm_ = 0; + uint32_t block_count_ = 0; + const uint8_t* block_headers_ = nullptr; + const uint8_t* block_data_ = nullptr; }; } // namespace doris::snii::format diff --git a/be/src/storage/index/snii/format/null_bitmap.cpp b/be/src/storage/index/snii/format/null_bitmap.cpp index 9e7d0d298f99f8..1cc12083fb08b0 100644 --- a/be/src/storage/index/snii/format/null_bitmap.cpp +++ b/be/src/storage/index/snii/format/null_bitmap.cpp @@ -111,10 +111,24 @@ NullBitmapWriter::~NullBitmapWriter() = default; void NullBitmapWriter::add_null(uint32_t docid) { bitmap_->add(docid); + optimized_ = false; } void NullBitmapWriter::add_many(std::span docids) { bitmap_->addMany(docids.size(), docids.data()); + optimized_ = false; +} + +void NullBitmapWriter::optimize_for_serialization() { + if (optimized_) { + return; + } + // NULL rows usually come in long runs (whole batches, absent VARIANT paths). + // Without run containers every container denser than 4096 values stays an + // 8 KiB bitset no matter how few runs it holds. + bitmap_->runOptimize(); + bitmap_->shrinkToFit(); + optimized_ = true; } uint32_t NullBitmapWriter::null_count() const { @@ -173,15 +187,25 @@ uint64_t NullBitmapWriter::build_memory_upper_bound(std::span so dense_container_count * (kDenseArrayConversionBytes + kBitsetBytes + sizeof(roaring::internal::array_container_t) + sizeof(roaring::internal::bitset_container_t)); - return sizeof(roaring::Roaring) + top_array_peak + sparse_peak + dense_peak; + // runOptimize converts one container at a time and frees the original after + // building its replacement; shrinkToFit reallocates one container at a time. + // A replacement is never larger than a bitset container, and the top-level + // shrink overlap is inside top_array_peak. + constexpr uint64_t kReplacementContainerPeak = + kBitsetBytes + std::max({sizeof(roaring::internal::array_container_t), + sizeof(roaring::internal::bitset_container_t), + sizeof(roaring::internal::run_container_t)}); + return sizeof(roaring::Roaring) + top_array_peak + sparse_peak + dense_peak + + kReplacementContainerPeak; } Status NullBitmapWriter::serialization_sizes(uint32_t doc_count, - NullBitmapSerializationSizes* out) const { + NullBitmapSerializationSizes* out) { if (out == nullptr) { return Status::Error( "null bitmap: null serialization size output"); } + optimize_for_serialization(); const size_t roaring_bytes = bitmap_->getSizeInBytes(); const size_t prefix_bytes = varint_len(doc_count) + varint_len(roaring_bytes); if (roaring_bytes > std::numeric_limits::max() - prefix_bytes) { @@ -200,7 +224,7 @@ Status NullBitmapWriter::serialization_sizes(uint32_t doc_count, return Status::OK(); } -Status NullBitmapWriter::finish(uint32_t doc_count, ByteSink* sink) const { +Status NullBitmapWriter::finish(uint32_t doc_count, ByteSink* sink) { if (sink == nullptr) { return Status::Error("null bitmap: null output sink"); } diff --git a/be/src/storage/index/snii/format/null_bitmap.h b/be/src/storage/index/snii/format/null_bitmap.h index a975a76d399d5d..4415f3d2896916 100644 --- a/be/src/storage/index/snii/format/null_bitmap.h +++ b/be/src/storage/index/snii/format/null_bitmap.h @@ -53,7 +53,9 @@ struct NullBitmapSerializationSizes { // On-disk layout (the whole section is framed by SectionFramer, which adds a // type + varint64 len + payload + fixed32 crc32c envelope): // framer payload = [varint64 doc_count][varint64 roaring_size][roaring_bytes] -// roaring_bytes is the portable CRoaring serialization (Roaring::write). +// roaring_bytes is the portable CRoaring serialization (Roaring::write) of the +// run-optimized bitmap. Run containers are part of the portable format, so every +// reader (including those that predate run optimization here) decodes them. class NullBitmapWriter { public: NullBitmapWriter(); @@ -70,19 +72,27 @@ class NullBitmapWriter { uint32_t null_count() const; // Conservative pre-allocation charge for constructing CRoaring from sorted - // docids. It includes top-level array growth and array-to-bitset conversion - // overlap, so the caller must retain this charge until the bitmap is destroyed. + // docids and run-optimizing it for serialization. It includes top-level array + // growth, array-to-bitset conversion overlap and the one replacement container + // run optimization or shrinking holds next to the container it replaces, so the + // caller must retain this charge until the bitmap is destroyed. static uint64_t build_memory_upper_bound(std::span sorted_docids); - Status serialization_sizes(uint32_t doc_count, NullBitmapSerializationSizes* out) const; + // Both serialization calls first convert the bitmap to its smallest container + // mix (runOptimize + shrinkToFit), once per batch of added docids, so the sizes + // reported here are the bytes finish() writes. + Status serialization_sizes(uint32_t doc_count, NullBitmapSerializationSizes* out); // Serializes [doc_count][roaring_size][roaring_bytes] framed by SectionFramer // and appends it to sink (does not clear sink). doc_count is the total number // of docs in the logical index (recorded so the reader can round-trip it). - Status finish(uint32_t doc_count, ByteSink* sink) const; + Status finish(uint32_t doc_count, ByteSink* sink); private: + void optimize_for_serialization(); + std::unique_ptr bitmap_; + bool optimized_ = true; }; // Read-only view: on open, SectionFramer verifies the CRC and truncation; this diff --git a/be/src/storage/index/snii/reader/logical_index_reader.cpp b/be/src/storage/index/snii/reader/logical_index_reader.cpp index 5f146397d667ac..690e4cfffb46f8 100644 --- a/be/src/storage/index/snii/reader/logical_index_reader.cpp +++ b/be/src/storage/index/snii/reader/logical_index_reader.cpp @@ -29,7 +29,6 @@ #include "storage/index/snii/encoding/byte_source.h" #include "storage/index/snii/encoding/crc32c.h" #include "storage/index/snii/encoding/section_framer.h" -#include "storage/index/snii/encoding/varint.h" #include "storage/index/snii/encoding/zstd_codec.h" #include "storage/index/snii/format/dict_block.h" #include "storage/index/snii/format/dict_block_directory.h" @@ -123,12 +122,12 @@ Status validate_norms_region(io::FileReader* reader, const RegionRef& norms, uin return Status::Error( "logical_index: norms doc count exceeds uint32"); } - const uint64_t payload_length = varint_len(doc_count) + doc_count; - const uint64_t expected_length = - 1 + varint_len(payload_length) + payload_length + sizeof(uint32_t); - if (norms.length != expected_length) { + // A dense section has exactly this length and a sparse one is never longer (the writer + // only picks it when it is shorter), which bounds the cache charge below by doc_count. + // NormsPodReader::open validates the exact layout when the section is loaded. + if (norms.length > format::dense_norms_section_bytes(doc_count)) { return Status::Error( - "logical_index: norms region length mismatch"); + "logical_index: norms region longer than a dense norms section"); } if (norms.length > std::numeric_limits::max()) { return Status::Error( diff --git a/be/src/storage/index/snii/reader/logical_index_reader.h b/be/src/storage/index/snii/reader/logical_index_reader.h index ee1e90b2571687..9460a31ebaaa1f 100644 --- a/be/src/storage/index/snii/reader/logical_index_reader.h +++ b/be/src/storage/index/snii/reader/logical_index_reader.h @@ -183,11 +183,12 @@ class LogicalIndexReader { LogicalIndexOpenMode open_mode() const { return open_mode_; } io::FileReader* reader() const { return reader_; } - // Returns a reader over the validated norms section. The first call reads - // and validates the section; later calls share the immutable reader-owned - // bytes. The full on-disk section is reserved in memory_usage() before this - // LogicalIndexReader enters the searcher cache, so lazy loading cannot make - // the cache under-report its eventual resident size. + // Returns a reader over the validated norms section (dense or sparse). The + // first call reads and validates the section; later calls share the immutable + // reader-owned bytes, which are all the reader needs. The full on-disk section + // is reserved in memory_usage() before this LogicalIndexReader enters the + // searcher cache, so lazy loading cannot make the cache under-report its + // eventual resident size. Status open_norms(format::NormsPodReader* out) const; // Compaction scans one source norm vector at a time. This charge matches the // reader's full cache accounting; release_compaction_norms() drops the loaded diff --git a/be/src/storage/index/snii/snii_index_writer.cpp b/be/src/storage/index/snii/snii_index_writer.cpp index 5d6d3094266fa6..53a42c22670576 100644 --- a/be/src/storage/index/snii/snii_index_writer.cpp +++ b/be/src/storage/index/snii/snii_index_writer.cpp @@ -37,7 +37,13 @@ #include "storage/tablet/tablet_schema.h" namespace doris::segment_v2 { -namespace {} // namespace +namespace { + +uint8_t saturated_norm_length(uint64_t token_count) { + return static_cast(std::min(token_count, 255)); +} + +} // namespace SniiIndexColumnWriter::SniiIndexColumnWriter(IndexFileWriter* index_file_writer, const TabletIndex* index_meta, FieldType value_type) @@ -217,7 +223,7 @@ Status SniiIndexColumnWriter::add_values(const std::string /*name*/, const void* uint32_t token_count = 0; RETURN_IF_ERROR(_add_value_tokens(*v, _rid, 0, &max_position, &token_count)); if (_writes_norms) { - _encoded_norms.push_back(::doris::snii::query::encode_norm(token_count)); + _norm_lengths.push_back(saturated_norm_length(token_count)); _report_encoded_norms_capacity(); } ++v; @@ -256,8 +262,9 @@ Status SniiIndexColumnWriter::add_array_values(size_t field_size, const void* va } if (_writes_norms) { // An ARRAY row's document length is the total token count across its elements. - // NULL rows also pass here with length 0 and are marked by add_array_nulls. - _encoded_norms.push_back(::doris::snii::query::encode_norm(row_token_count)); + // NULL rows also pass here; add_array_nulls drops the entries of those without + // tokens. + _norm_lengths.push_back(saturated_norm_length(row_token_count)); _report_encoded_norms_capacity(); } start_off += array_elem_size; @@ -270,8 +277,10 @@ void SniiIndexColumnWriter::_report_null_docids_capacity(bool release_all) { if (_memory_reporter == nullptr) { return; } - const int64_t now = - release_all ? 0 : static_cast(_null_docids.capacity() * sizeof(uint32_t)); + const int64_t now = release_all ? 0 + : static_cast((_null_docids.capacity() + + _null_docids_with_norms.capacity()) * + sizeof(uint32_t)); if (now != _null_docids_charged_bytes) { _memory_reporter->report(now - _null_docids_charged_bytes); _null_docids_charged_bytes = now; @@ -282,7 +291,7 @@ void SniiIndexColumnWriter::_report_encoded_norms_capacity(bool release_all) { if (_memory_reporter == nullptr) { return; } - const int64_t now = release_all ? 0 : static_cast(_encoded_norms.capacity()); + const int64_t now = release_all ? 0 : static_cast(_norm_lengths.capacity()); if (now != _encoded_norms_charged_bytes) { _memory_reporter->report(now - _encoded_norms_charged_bytes); _encoded_norms_charged_bytes = now; @@ -311,10 +320,7 @@ Status SniiIndexColumnWriter::add_nulls(uint32_t count) { _null_docids.push_back(_rid + i); } _rid += count; - if (_writes_norms) { - _encoded_norms.insert(_encoded_norms.end(), count, ::doris::snii::query::encode_norm(0)); - _report_encoded_norms_capacity(); - } + // A NULL scalar row produces no token, so it carries no norm. _report_null_docids_capacity(); return Status::OK(); } @@ -328,11 +334,35 @@ Status SniiIndexColumnWriter::add_array_nulls(const uint8_t* null_map, size_t nu return Status::OK(); } const auto first_row = _rid - num_rows; + if (!_writes_norms) { + for (size_t i = 0; i < num_rows; ++i) { + if (null_map[i] == 1) { + _null_docids.push_back(cast_set(first_row + i)); + } + } + _report_null_docids_capacity(); + return Status::OK(); + } + // add_array_values appended one length for each of these rows. Drop the NULL rows that + // produced no token; a NULL row with tokens keeps its norm, because its tokens are in + // the postings and scoring reads the norm of every posting document. + DORIS_CHECK_EQ(_norm_lengths.size() + _null_docids.size(), + _rid + _null_docids_with_norms.size()); + const size_t batch_begin = _norm_lengths.size() - num_rows; + size_t kept = batch_begin; for (size_t i = 0; i < num_rows; ++i) { + const uint8_t length = _norm_lengths[batch_begin + i]; if (null_map[i] == 1) { - _null_docids.push_back(cast_set(first_row + i)); + const auto docid = cast_set(first_row + i); + _null_docids.push_back(docid); + if (length == 0) { + continue; + } + _null_docids_with_norms.push_back(docid); } + _norm_lengths[kept++] = length; } + _norm_lengths.resize(kept); _report_null_docids_capacity(); return Status::OK(); } @@ -353,8 +383,14 @@ Status SniiIndexColumnWriter::finish() { IndexFileWriter::SniiAddIndexOptions options {}; options.is_direct_load = _is_direct_load; if (_writes_norms) { - DORIS_CHECK_EQ(_encoded_norms.size(), _rid); - options.encoded_norms = std::move(_encoded_norms); + DORIS_CHECK_EQ(_norm_lengths.size() + _null_docids.size(), + _rid + _null_docids_with_norms.size()); + for (uint8_t& length : _norm_lengths) { + length = ::doris::snii::query::encode_norm(length); + } + options.write_norms = true; + options.encoded_norms = std::move(_norm_lengths); + options.null_docids_with_norms = std::move(_null_docids_with_norms); } status = _index_file_writer->add_snii_index( _index_meta, cast_set(_rid), std::move(_null_docids), _term_buffer.get(), @@ -381,7 +417,8 @@ void SniiIndexColumnWriter::close_on_error() { _report_encoded_norms_capacity(/*release_all=*/true); _memory_reporter.reset(); _null_docids.clear(); - std::vector().swap(_encoded_norms); + std::vector().swap(_null_docids_with_norms); + std::vector().swap(_norm_lengths); } } // namespace doris::segment_v2 diff --git a/be/src/storage/index/snii/snii_index_writer.h b/be/src/storage/index/snii/snii_index_writer.h index ed54a4657bb4d2..844b49fb4c04f8 100644 --- a/be/src/storage/index/snii/snii_index_writer.h +++ b/be/src/storage/index/snii/snii_index_writer.h @@ -68,7 +68,10 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { ::doris::snii::writer::MemoryReporter* memory_reporter_for_test() const { return _memory_reporter.get(); } - const std::vector& encoded_norms_for_test() const { return _encoded_norms; } + const std::vector& norm_lengths_for_test() const { return _norm_lengths; } + const std::vector& null_docids_with_norms_for_test() const { + return _null_docids_with_norms; + } ::doris::snii::format::IndexConfig config_for_test() const { return _config; } bool writes_norms_for_test() const { return _writes_norms; } void set_analysis_for_test(inverted_index::ReaderPtr reader, @@ -114,9 +117,16 @@ class SniiIndexColumnWriter final : public IndexColumnWriter { std::unique_ptr<::doris::snii::writer::MemoryReporter> _memory_reporter; std::unique_ptr<::doris::snii::writer::SpimiTermBuffer> _term_buffer; std::vector _null_docids; - std::vector _encoded_norms; - // Bytes of _null_docids capacity currently mirrored into _memory_reporter - // (and through it the SNII index-build observation tracker). Re-charged on + // Norms are only kept for the rows that carry one (see format/norms_pod.h): every + // non-NULL row, plus the NULL ARRAY rows that still produced tokens, which are listed + // in _null_docids_with_norms. _norm_lengths holds their token counts saturated at 255, + // in row order; finish() encodes them. Keeping the raw count until the row's NULL flag + // is known (add_array_nulls runs after add_array_values) is what separates an empty + // NULL row from a NULL row with one token -- both encode to the same byte. + std::vector _norm_lengths; + std::vector _null_docids_with_norms; + // Bytes of _null_docids (+ _null_docids_with_norms) capacity currently mirrored into + // _memory_reporter (and through it the SNII index-build observation tracker). Re-charged on // growth in add_nulls / add_array_nulls, released in finish() / close_on_error() -- // without it a large interleaved-null segment accumulates untracked RSS the // G09 limiter cannot see. diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.cpp b/be/src/storage/index/snii/stats/snii_stats_provider.cpp index 557c9c30417da7..e8cb5e6043b851 100644 --- a/be/src/storage/index/snii/stats/snii_stats_provider.cpp +++ b/be/src/storage/index/snii/stats/snii_stats_provider.cpp @@ -66,6 +66,13 @@ Status SniiStatsProvider::open(const reader::LogicalIndexReader* idx, SniiStatsP "snii_stats: norms doc count {} differs from segment doc count {}", out->norms_reader_.doc_count(), sb.doc_count); } + // Every non-NULL document carries a norm; a sparse section may also cover NULL documents + // that produced tokens. + if (out->norms_reader_.present_count() < sb.indexed_doc_count) { + return Status::Error( + "snii_stats: norms cover {} documents, fewer than the {} indexed documents", + out->norms_reader_.present_count(), sb.indexed_doc_count); + } out->has_norms_ = true; return Status::OK(); } diff --git a/be/src/storage/index/snii/stats/snii_stats_provider.h b/be/src/storage/index/snii/stats/snii_stats_provider.h index 28ef31c597c582..8944cefc68cb2d 100644 --- a/be/src/storage/index/snii/stats/snii_stats_provider.h +++ b/be/src/storage/index/snii/stats/snii_stats_provider.h @@ -30,9 +30,10 @@ // sum_total_term_freq). // - per-term df from the term's DictEntry (resolved through the reader's // lookup flow). -// - per-doc length normalization byte (encoded_norm) from the norms POD, -// lazily loaded and validated once by LogicalIndexReader, then shared by -// every stats provider for that cached logical index. +// - per-doc length normalization byte (encoded_norm) from the norms section +// (dense or sparse, see format/norms_pod.h), lazily loaded and validated once +// by LogicalIndexReader, then shared by every stats provider for that cached +// logical index. // // avgdl() = sum_total_term_freq / max(1, indexed_doc_count): the average document // length used by BM25 length normalization. The provider performs no scoring; it @@ -59,8 +60,11 @@ class SniiStatsProvider { // Per-term document frequency. Absent term -> *df = 0 (OK status). Status doc_freq(std::string_view term, uint64_t* df) const; - // 1-byte encoded doc-length norm for docid (raw byte from the norms POD). - // Out-of-range docid -> InvalidArgument; index without norms -> InvalidArgument. + // 1-byte encoded doc-length norm for docid (raw byte from the norms section). + // Out-of-range docid -> InvalidArgument; index without norms -> InvalidArgument; + // a docid without a norm in a sparse section (never one read from a posting of + // a valid index) -> Corruption. A sparse lookup costs O(log blocks + log block + // size), a dense one O(1). Status encoded_norm(uint32_t docid, uint8_t* out) const; bool has_norms() const { return has_norms_; } diff --git a/be/src/storage/index/snii/writer/logical_index_writer.cpp b/be/src/storage/index/snii/writer/logical_index_writer.cpp index 9aec2a54a95c17..a4b95526ad0fb9 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.cpp +++ b/be/src/storage/index/snii/writer/logical_index_writer.cpp @@ -24,9 +24,9 @@ #include #include +#include "common/config.h" #include "storage/index/snii/common/slice.h" #include "storage/index/snii/encoding/crc32c.h" -#include "storage/index/snii/encoding/varint.h" #include "storage/index/snii/encoding/zstd_codec.h" #include "storage/index/snii/format/bsbf.h" #include "storage/index/snii/format/dict_block.h" @@ -416,6 +416,7 @@ LogicalIndexWriter::LogicalIndexWriter(const SniiIndexInput& in, TrackedNullDoci terms_(in.terms), term_source_(in.term_source), encoded_norms_(in.encoded_norms), + null_docids_with_norms_(in.null_docids_with_norms), target_dict_block_bytes_(in.target_dict_block_bytes != 0 ? in.target_dict_block_bytes : format::kDefaultTargetDictBlockBytes), @@ -585,9 +586,23 @@ Status LogicalIndexWriter::prepare_build(io::FileWriter* posting_out) { } Status LogicalIndexWriter::finalize_build() { - if (has_norms_ && encoded_norms_.size() != doc_count_) { + if (!has_norms_ && !null_docids_with_norms_.empty()) { return Status::Error( - "logical_index: norms length must equal doc_count"); + "logical_index: NULL docids with norms require norms"); + } + const format::NormsSectionInput norms_input { + .doc_count = doc_count_, + .null_docids = std::span(null_docids_.data(), null_docids_.size()), + .null_docids_with_norms = null_docids_with_norms_, + .norms = encoded_norms_}; + format::NormsSectionPlan norms_plan; + if (has_norms_) { + // The only place that picks the norms layout: loads, compaction output (streamed + // sessions included) and index rebuilds all finish their logical indexes here. With + // enable_snii_sparse_norms off, the section is the dense one every earlier writer + // produced, so BEs without sparse-norms support can read it. + const bool force_dense_norms = !config::enable_snii_sparse_norms; + RETURN_IF_ERROR(format::plan_norms_section(norms_input, force_dense_norms, &norms_plan)); } // Seal the dict buffer so a spilled temp is flushed before // stream_dict_region_into reads it back. A no-op for a RAM-resident dict. @@ -599,19 +614,17 @@ Status LogicalIndexWriter::finalize_build() { stats_.null_count = static_cast(null_docids_.size()); if (has_norms_) { - const size_t payload_size = varint_len(encoded_norms_.size()) + encoded_norms_.size(); - const size_t section_size = 1 + varint_len(payload_size) + payload_size + sizeof(uint32_t); MemoryReporter::Reservation build_reservation = memory_reporter_ == nullptr ? MemoryReporter::Reservation() : memory_reporter_->make_reservation(); if (memory_reporter_ != nullptr) { - RETURN_IF_ERROR(build_reservation.set_bytes(payload_size)); - RETURN_IF_ERROR(norms_section_reservation_.set_bytes(section_size)); + RETURN_IF_ERROR(build_reservation.set_bytes(norms_plan.payload_bytes)); + RETURN_IF_ERROR(norms_section_reservation_.set_bytes(norms_plan.framed_bytes)); } ByteSink nsink; - format::NormsPodWriter::finish(encoded_norms_, &nsink); + format::write_norms_section(norms_input, norms_plan, &nsink); norms_section_ = nsink.take(); - DORIS_CHECK_EQ(norms_section_.capacity(), section_size); + DORIS_CHECK_EQ(norms_section_.capacity(), norms_plan.framed_bytes); if (memory_reporter_ != nullptr) { DORIS_CHECK_EQ(norms_section_reservation_.bytes(), norms_section_.capacity()); } diff --git a/be/src/storage/index/snii/writer/logical_index_writer.h b/be/src/storage/index/snii/writer/logical_index_writer.h index 693d020cc6ede0..da32dfae11c1d9 100644 --- a/be/src/storage/index/snii/writer/logical_index_writer.h +++ b/be/src/storage/index/snii/writer/logical_index_writer.h @@ -103,11 +103,19 @@ struct SniiIndexInput { format::IndexConfig config = format::IndexConfig::kDocsPositions; uint32_t doc_count = 0; std::vector null_docids; - // Per-doc 1-byte encoded norm (length doc_count); only consumed when the - // config has scoring. May be empty otherwise. + // 1-byte encoded norms of the documents that carry one, in ascending docid + // order: every document outside null_docids plus null_docids_with_norms + // (see format/norms_pod.h). Only consumed when the index writes norms; may be + // empty otherwise. std::vector encoded_norms; - // Streaming merge sessions declare norms up front but supply them only before finish, - // after rebuilding them alongside postings. The writer validates their size at finalize. + // Ascending subset of null_docids whose documents still produced tokens and + // therefore carry a norm: a nullable ARRAY row can keep its nested payload + // under the NULL flag, and its tokens are indexed like any other row's. + std::vector null_docids_with_norms; + // The index writes a norms section. Implied by a nonempty encoded_norms; + // required when every document is NULL (no norm to hand over). Streaming merge + // sessions declare norms up front but supply them only before finish, after + // rebuilding them alongside postings. The writer validates their size at finalize. bool write_norms = false; // G16-h: zstd levels for the dict-block whole-block compression and the // .prx window auto mode (both default 3 == the historical constants). @@ -176,6 +184,7 @@ class TrackedNullDocids { } private: + friend class SniiStreamedIndexSession; MemoryReporter::Reservation reservation_; std::vector docids_; }; @@ -273,7 +282,8 @@ class LogicalIndexWriter { // and spills to a temp once it crosses the RAM cap (bounded peak RSS for a huge // dict). Its bytes are emitted via stream_dict_region_into below. The posting region // went straight to the output during build(), so it has no length accessor here -- - // the orchestrator measures it directly. norms stays in RAM (1 byte/doc). + // the orchestrator measures it directly. The norms section stays in RAM (at most + // 1 byte/doc, see format/norms_pod.h). uint64_t dict_region_size() const { return dict_buf_.size(); } const std::vector& norms_bytes() const { return norms_section_; } const std::vector& null_bitmap_bytes() const { return null_bitmap_section_; } @@ -359,6 +369,7 @@ class LogicalIndexWriter { SpimiTermBuffer* term_source_; // streaming source (null => use terms_) uint64_t term_count_ = 0; // distinct terms actually consumed const std::vector& encoded_norms_; + const std::vector& null_docids_with_norms_; uint32_t target_dict_block_bytes_; // G16-h: zstd levels (dict whole-block / prx auto mode), from SniiIndexInput. diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.cpp b/be/src/storage/index/snii/writer/snii_compound_writer.cpp index 89256f88dd7272..3c60d4d59a84c9 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.cpp +++ b/be/src/storage/index/snii/writer/snii_compound_writer.cpp @@ -18,6 +18,7 @@ #include "storage/index/snii/writer/snii_compound_writer.h" #include +#include #include #include @@ -355,7 +356,8 @@ Status SniiStreamedIndexSession::push_term(StreamedTermPostings&& tp) { return Status::OK(); } -Status SniiStreamedIndexSession::set_encoded_norms(TrackedEncodedNorms encoded_norms) { +Status SniiStreamedIndexSession::set_encoded_norms(TrackedEncodedNorms encoded_norms, + TrackedNullDocids null_docids_with_norms) { if (!owner_->failed_.ok()) return owner_->failed_; if (finished_) { return Status::Error( @@ -369,18 +371,28 @@ Status SniiStreamedIndexSession::set_encoded_norms(TrackedEncodedNorms encoded_n return Status::Error( "compound: norms were already set"); } - if (encoded_norms.size() != input_.doc_count) { + const uint64_t norm_documents = uint64_t {input_.doc_count} - writer_->null_docids_.size() + + null_docids_with_norms.size(); + if (encoded_norms.size() != norm_documents) { return Status::Error( - "compound: norms length {} differs from doc_count {}", encoded_norms.size(), - input_.doc_count); + "compound: norms length {} differs from the {} documents that carry a norm", + encoded_norms.size(), norm_documents); } - // writer_ references input_.encoded_norms; move into it here for finalize to read by reference. + // writer_ references input_.encoded_norms and input_.null_docids_with_norms; move into them + // here for finalize to read by reference. encoded_norms_reservation_ = std::move(encoded_norms.reservation_); input_.encoded_norms = std::move(encoded_norms.norms_); + null_docids_with_norms_reservation_ = std::move(null_docids_with_norms.reservation_); + input_.null_docids_with_norms = std::move(null_docids_with_norms.docids_); norms_set_ = true; return Status::OK(); } +std::span SniiStreamedIndexSession::null_docids() const { + DORIS_CHECK(writer_ != nullptr); + return {writer_->null_docids_.data(), writer_->null_docids_.size()}; +} + Status SniiStreamedIndexSession::finish() { if (!owner_->failed_.ok()) return owner_->failed_; if (finished_) { @@ -439,9 +451,10 @@ Status SniiCompoundWriter::begin_streamed_index(SniiIndexInput in, TrackedNullDo if (!in.null_docids.empty()) return Status::Error( "compound: tracked streamed NULL docids must not also be present in input"); - if (!in.encoded_norms.empty()) + if (!in.encoded_norms.empty() || !in.null_docids_with_norms.empty()) { return Status::Error( "compound: tracked streamed norms must not also be present in input"); + } RETURN_IF_ERROR(ensure_bootstrap()); auto s = std::unique_ptr( new SniiStreamedIndexSession(this, std::move(in), std::move(null_docids))); @@ -461,6 +474,8 @@ Status SniiCompoundWriter::finish_streamed_index(SniiStreamedIndexSession* sessi // its transferred charge before retaining that section for compound finish. std::vector().swap(session->input_.encoded_norms); session->encoded_norms_reservation_.reset(); + std::vector().swap(session->input_.null_docids_with_norms); + session->null_docids_with_norms_reservation_.reset(); Placement p; p.post_off = session->post_off_; p.post_len = out_->bytes_written() - p.post_off; @@ -501,9 +516,33 @@ Status SniiCompoundWriter::write_index_aux_sections(LogicalIndexWriter& writer, writer.release_norms_bytes(); } if (writer.has_null_bitmap()) { - placement.null_off = out_->bytes_written(); - RETURN_IF_ERROR(append(writer.null_bitmap_bytes())); - placement.null_len = out_->bytes_written() - placement.null_off; + // A VARIANT column copies each index definition to every subcolumn, and all + // definitions on one subcolumn are written back to back under the same suffix + // with the same NULL rows, so they produce byte-identical bitmaps. When this + // bitmap equals the last one written and carries the same suffix, point this + // index at that region instead of storing the bytes again. Region references + // are absolute, so the format is unchanged; the bitmap records the doc count, so + // equal bytes also mean an equal document domain. + // + // The earlier bitmap's bytes were released as soon as they reached the file, so + // a 128-bit hash plus the length stands in for comparing the bytes themselves. + const std::vector& bytes = writer.null_bitmap_bytes(); + const XXH128_hash_t hash = XXH3_128bits(bytes.data(), bytes.size()); + WrittenNullBitmap& last = last_null_bitmap_; + if (last.length == bytes.size() && last.hash_low64 == hash.low64 && + last.hash_high64 == hash.high64 && last.index_suffix == writer.index_suffix()) { + placement.null_off = last.offset; + placement.null_len = last.length; + } else { + placement.null_off = out_->bytes_written(); + RETURN_IF_ERROR(append(bytes)); + placement.null_len = out_->bytes_written() - placement.null_off; + last = {.index_suffix = writer.index_suffix(), + .hash_low64 = hash.low64, + .hash_high64 = hash.high64, + .offset = placement.null_off, + .length = placement.null_len}; + } writer.release_null_bitmap_bytes(); } if (writer.has_bsbf()) { diff --git a/be/src/storage/index/snii/writer/snii_compound_writer.h b/be/src/storage/index/snii/writer/snii_compound_writer.h index cb3092d780db81..f41c8d11515919 100644 --- a/be/src/storage/index/snii/writer/snii_compound_writer.h +++ b/be/src/storage/index/snii/writer/snii_compound_writer.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -50,8 +51,10 @@ class SniiRewriteSnapshot; // [DICT blocks region] concatenated DICT blocks, split by // target_dict_block_bytes // for each logical index, in add order: -// [norms POD] NormsPodWriter::finish (scoring only; else absent) -// [null bitmap POD] NullBitmapWriter::finish (when nulls exist) +// [norms] write_norms_section, dense or sparse (scoring only; else absent) +// [null bitmap POD] NullBitmapWriter::finish (when nulls exist and no earlier index +// with the same suffix wrote the same bitmap; such an index +// references that region instead, see write_index_aux_sections) // for each logical index, in add order: // [Core metadata][SampledTermIndex blob][DICT block directory blob] // [metadata directory] raw SniiMetadataDirectoryPB bytes @@ -73,7 +76,9 @@ class SniiRewriteSnapshot; // - SectionRefs in each Core metadata record ABSOLUTE file offset+length of // that index's posting, DICT, norms, null-bitmap, and BSBF regions. Absent // regions are (0,0); a present-but-empty posting region (all-INLINE index) -// is (off, 0). +// is (off, 0). Indexes with the same suffix and byte-identical null bitmaps +// reference one shared null-bitmap region; every other region belongs to +// exactly one index. // - DictBlockDirectory entries record each DICT block's ABSOLUTE file offset + // length. // - A windowed/slim pod_ref entry's absolute .frq offset = @@ -132,9 +137,14 @@ class SniiStreamedIndexSession { // entered the compound output; all later calls return the first error. Status push_term(StreamedTermPostings&& tp); // Supply this destination segment's norms, rebuilt alongside postings during compaction. - // Sessions declaring write_norms must call this exactly once before finish, with doc_count - // entries. - Status set_encoded_norms(TrackedEncodedNorms encoded_norms); + // Sessions declaring write_norms must call this exactly once before finish, with one entry + // per document that carries a norm: the documents outside null_docids() plus + // null_docids_with_norms (an ascending subset of null_docids(), see SniiIndexInput). + Status set_encoded_norms( + TrackedEncodedNorms encoded_norms, + TrackedNullDocids null_docids_with_norms = TrackedNullDocids(std::vector())); + // The NULL docids this session was begun with. Valid until finish(). + std::span null_docids() const; // Seals this index: flushes the trailing DICT block, streams the DICT region // right after the posting region and records the placements. A failed finish // leaves the session unfinished (and the container unsealable) -- there is @@ -152,9 +162,10 @@ class SniiStreamedIndexSession { TrackedNullDocids null_docids); SniiCompoundWriter* owner_; - // The reservation precedes input_ so input_.encoded_norms is destroyed - // before its charge is released. + // The reservations precede input_ so input_.encoded_norms and + // input_.null_docids_with_norms are destroyed before their charges are released. MemoryReporter::Reservation encoded_norms_reservation_; + MemoryReporter::Reservation null_docids_with_norms_reservation_; // Owns the input: LogicalIndexWriter keeps references into it (terms / // encoded_norms), so it must live exactly as long as the writer. SniiIndexInput input_; @@ -258,6 +269,16 @@ class SniiCompoundWriter { size_t dict_block_directory_length = 0; }; + // A null-bitmap section this writer appended: the suffix of the index that wrote + // it, a 128-bit hash of its framed bytes, and where they landed. + struct WrittenNullBitmap { + std::string index_suffix; + uint64_t hash_low64 = 0; + uint64_t hash_high64 = 0; + uint64_t offset = 0; + uint64_t length = 0; + }; + // One registered blob logical index awaiting finish(). cold/hot refs are // resolved as the corresponding bytes stream out during finish(). struct PendingBlobIndex { @@ -278,6 +299,8 @@ class SniiCompoundWriter { // [posting][dict] pair and fills its placement. Keeping one index's sections // contiguous is what makes a single-index cold query touch one cache block instead // of three; the previous layout grouped these by section type across all indexes. + // The one exception is a null bitmap already written for the same suffix, which + // is referenced rather than written again (see the .cpp). Status write_index_aux_sections(LogicalIndexWriter& writer, Placement& placement); Status write_tail(); Status append(const std::vector& bytes); @@ -331,6 +354,9 @@ class SniiCompoundWriter { // Blob logical indexes registered by add_blob_index(), in add order. Their // bytes stream out during finish() only. std::vector blobs_; + // The last null bitmap appended (length 0 until the first one): the region the next + // index on the same suffix references when its bitmap is identical. + WrittenNullBitmap last_null_bitmap_; // inherit() ran successfully. Distinct from inherited_ being non-empty: a // rewrite may drop every old index and still copy the bootstrap header. bool inherited_prefix_ = false; diff --git a/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp b/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp index 539e49e49e2225..f6c11e3dd3e585 100644 --- a/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/boolean_query_test.cpp @@ -385,7 +385,7 @@ TEST_F(BooleanQueryTest, test_boolean_query_scoring_or) { std::wstring ws_field = StringHelper::to_wstring("name1"); // 直接访问成员填充统计信息 - context->collection_statistics->_total_num_docs = 80; + context->collection_statistics->_total_num_docs[ws_field] = 80; context->collection_statistics->_total_num_tokens[ws_field] = 240; // 80*3 auto set_df = [&](const std::string& term, uint64_t df) { context->collection_statistics->_term_doc_freqs[ws_field][StringHelper::to_wstring(term)] = diff --git a/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp b/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp index 2d8b34b2bb7a77..3c21e54a34cb61 100644 --- a/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/multi_phrase_query_test.cpp @@ -498,7 +498,7 @@ TEST_F(MultiPhraseQueryV2Test, test_multi_phrase_query_with_scoring) { term_infos.push_back(term2); // Fill collection statistics for scoring - context->collection_statistics->_total_num_docs = reader_holder->numDocs(); + context->collection_statistics->_total_num_docs[field] = reader_holder->numDocs(); context->collection_statistics->_total_num_tokens[field] = reader_holder->numDocs() * 8; context->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("quick")] = 10; context->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("fast")] = 5; @@ -757,7 +757,7 @@ TEST_F(MultiPhraseQueryV2Test, test_multi_phrase_query_bm25_similarity) { term_infos.push_back(term3); // Setup statistics for BM25 - context->collection_statistics->_total_num_docs = reader_holder->numDocs(); + context->collection_statistics->_total_num_docs[field] = reader_holder->numDocs(); context->collection_statistics->_total_num_tokens[field] = reader_holder->numDocs() * 8; context->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("quick")] = 10; context->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("fast")] = 5; diff --git a/be/test/storage/index/inverted/query_v2/phrase_prefix_query_test.cpp b/be/test/storage/index/inverted/query_v2/phrase_prefix_query_test.cpp index 4016008d631ec2..f854d0bae8ed27 100644 --- a/be/test/storage/index/inverted/query_v2/phrase_prefix_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/phrase_prefix_query_test.cpp @@ -332,7 +332,7 @@ TEST_F(PhrasePrefixQueryV2Test, scorer_with_scoring) { std::wstring field = StringHelper::to_wstring("content"); // Setup collection statistics for BM25 - ctx->collection_statistics->_total_num_docs = reader->numDocs(); + ctx->collection_statistics->_total_num_docs[field] = reader->numDocs(); ctx->collection_statistics->_total_num_tokens[field] = reader->numDocs() * 8; ctx->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("quick")] = 10; diff --git a/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp b/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp index b974daee6a073a..838e97225dd42e 100644 --- a/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp +++ b/be/test/storage/index/inverted/query_v2/phrase_query_test.cpp @@ -386,7 +386,7 @@ TEST_F(PhraseQueryV2Test, test_phrase_query_scoring) { } // Fill collection statistics for scoring - context->collection_statistics->_total_num_docs = reader_holder->numDocs(); + context->collection_statistics->_total_num_docs[field] = reader_holder->numDocs(); context->collection_statistics->_total_num_tokens[field] = reader_holder->numDocs() * 8; context->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("quick")] = 10; context->collection_statistics->_term_doc_freqs[field][StringHelper::to_wstring("brown")] = 10; @@ -621,7 +621,7 @@ TEST_F(PhraseQueryV2Test, test_phrase_query_bm25_similarity) { } // Setup statistics for BM25 - context->collection_statistics->_total_num_docs = reader_holder->numDocs(); + context->collection_statistics->_total_num_docs[field] = reader_holder->numDocs(); context->collection_statistics->_total_num_tokens[field] = reader_holder->numDocs() * 8; for (const auto& term : terms) { context->collection_statistics->_term_doc_freqs[field][term] = 5; diff --git a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp index f9c7693b67b09a..f89e08b77efe53 100644 --- a/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp +++ b/be/test/storage/index/inverted/similarity/collection_statistics_test.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -455,7 +456,9 @@ class CollectionStatisticsTest : public ::testing::Test { } // A normal analyzed SNII segment with positions and norms, as emitted for scoring indexes. - Status write_snii_scoring_segment(const std::string& segment_path) { + // With with_nulls, the segment has two extra NULL rows (docids 1 and 3); the postings and + // norms of the non-NULL rows stay the same. + Status write_snii_scoring_segment(const std::string& segment_path, bool with_nulls = false) { const std::string index_path_prefix { segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path)}; io::FileWriterPtr file_writer; @@ -485,6 +488,11 @@ class CollectionStatisticsTest : public ::testing::Test { input.config = snii::format::IndexConfig::kDocsPositions; input.doc_count = 2; input.encoded_norms = {snii::query::encode_norm(2), snii::query::encode_norm(1)}; + if (with_nulls) { + alpha.docids = {0, 2}; + input.doc_count = 4; + input.null_docids = {1, 3}; + } input.terms = {std::move(alpha), std::move(beta)}; RETURN_IF_ERROR(writer.add_logical_index(input)); @@ -628,15 +636,15 @@ class CollectionStatisticsTest : public ::testing::Test { } struct SniiScoringFieldInput { - SniiScoringFieldInput(std::wstring field_name, uint64_t index_doc_count, + SniiScoringFieldInput(std::wstring field_name, uint64_t indexed_doc_count, uint64_t sum_total_term_freq, bool has_norms = true) : field_name(std::move(field_name)), - index_doc_count(index_doc_count), + indexed_doc_count(indexed_doc_count), sum_total_term_freq(sum_total_term_freq), has_norms(has_norms) {} std::wstring field_name; - uint64_t index_doc_count = 0; + uint64_t indexed_doc_count = 0; uint64_t sum_total_term_freq = 0; bool has_positions = true; bool has_norms = true; @@ -647,7 +655,7 @@ class CollectionStatisticsTest : public ::testing::Test { CollectionStatistics::SniiScoringSegmentAccumulator* segment_accumulator) { for (const auto& field : fields) { RETURN_IF_ERROR(statistics->admit_snii_scoring_segment( - field.field_name, field.index_doc_count, field.sum_total_term_freq, + field.field_name, field.indexed_doc_count, field.sum_total_term_freq, field.has_positions, field.has_norms, segment_accumulator)); } return Status::OK(); @@ -662,10 +670,10 @@ class CollectionStatisticsTest : public ::testing::Test { } Status admit_snii_segment_for_test(CollectionStatistics* statistics, - const std::wstring& field_name, uint64_t index_doc_count, + const std::wstring& field_name, uint64_t indexed_doc_count, uint64_t sum_total_term_freq, bool has_norms = true) { return admit_snii_fields_for_test( - statistics, {{field_name, index_doc_count, sum_total_term_freq, has_norms}}); + statistics, {{field_name, indexed_doc_count, sum_total_term_freq, has_norms}}); } Status stage_snii_fields_then_file_not_found_for_test( @@ -684,7 +692,7 @@ class CollectionStatisticsTest : public ::testing::Test { void expect_collected_stats(const std::wstring& field_name, uint64_t doc_count, uint64_t token_count) { - EXPECT_EQ(stats_->get_doc_num(), doc_count); + EXPECT_EQ(stats_->get_doc_num(field_name), doc_count); expect_collected_tokens(field_name, token_count); } @@ -991,6 +999,30 @@ TEST_F(CollectionStatisticsTest, SniiScoringUsesPhysicalStatistics) { expect_collected_term(L"1", L"alpha", 2); } +// NULL rows are not documents of the field: N and the avgdl denominator use the indexed count. +TEST_F(CollectionStatisticsTest, SniiScoringCountsIndexedDocumentsPerField) { + auto tablet_schema = create_snii_schema(); + const std::string segment_path = test_dir_ + "/snii_scoring_nulls_0.dat"; + ASSERT_TRUE(write_snii_scoring_segment(segment_path, /*with_nulls=*/true).ok()); + + auto rowset_meta = std::make_shared(); + auto rowset = std::make_shared(tablet_schema, rowset_meta); + rowset->set_num_segments(1); + rowset->set_segment_path(0, segment_path); + auto reader = std::make_shared(rowset); + std::vector splits {RowSetSplits(reader)}; + + auto status = stats_->collect(runtime_state_.get(), splits, tablet_schema, + create_match_expr_contexts("alpha"), nullptr); + + ASSERT_TRUE(status.ok()) << status; + expect_collected_stats(L"1", 2, 3); + expect_collected_term(L"1", L"alpha", 2); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 1.5F); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_idf(L"1", L"alpha"), + static_cast(std::log(1 + (2 - 2 + 0.5) / (2 + 0.5)))); +} + TEST_F(CollectionStatisticsTest, SniiScoringLookupUsesCallerIoContext) { snii::snii_test::ScopedEnv force_nonresident_dict("SNII_DICT_RESIDENT_MAX", "0"); auto tablet_schema = create_snii_schema(); @@ -1127,7 +1159,9 @@ TEST_F(CollectionStatisticsTest, CollectWithMultipleRowsetSplits) { class TestableCollectionStatistics : public CollectionStatistics { public: - void set_total_num_docs(uint64_t num_docs) { _total_num_docs = num_docs; } + void set_total_num_docs(const std::wstring& field_name, uint64_t num_docs) { + _total_num_docs[field_name] = num_docs; + } void set_total_num_tokens(const std::wstring& field_name, uint64_t num_tokens) { _total_num_tokens[field_name] = num_tokens; @@ -1198,7 +1232,7 @@ TEST_F(CollectionStatisticsTest, SegmentWithoutNormsRejectsWholeCollection) { EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); expect_no_collected_tokens(L"1"); - EXPECT_THROW(stats_->get_doc_num(), Exception); + EXPECT_THROW(stats_->get_doc_num(L"1"), Exception); } TEST_F(CollectionStatisticsTest, SegmentsAccumulatePhysicalStatistics) { @@ -1219,13 +1253,51 @@ TEST_F(CollectionStatisticsTest, MultiFieldSegmentsCommitAndAccumulateAtomically EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"2"), 4.0F); } -TEST_F(CollectionStatisticsTest, MultiFieldSegmentDocCountsMustAgree) { - auto status = admit_snii_fields_for_test(stats_.get(), {{L"1", 3, 7}, {L"2", 4, 12}}); +// SNII fields of one segment count their own indexed (non-NULL) documents, so they may differ. +TEST_F(CollectionStatisticsTest, MultiFieldSegmentKeepsPerFieldDocCounts) { + ASSERT_TRUE(admit_snii_fields_for_test(stats_.get(), {{L"1", 3, 7}, {L"2", 4, 12}}).ok()); + ASSERT_TRUE(admit_snii_fields_for_test(stats_.get(), {{L"1", 1, 2}, {L"2", 0, 0}}).ok()); - EXPECT_EQ(status.code(), ErrorCode::INVERTED_INDEX_NOT_SUPPORTED); - expect_no_collected_tokens(L"1"); - expect_no_collected_tokens(L"2"); - EXPECT_THROW(stats_->get_doc_num(), Exception); + expect_collected_stats(L"1", 4, 9); + expect_collected_stats(L"2", 4, 12); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"1"), 9.0F / 4.0F); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"2"), 3.0F); +} + +// A field whose documents are all NULL in the whole collection still has finite statistics. +TEST_F(CollectionStatisticsTest, FieldWithoutIndexedDocumentsHasFiniteStatistics) { + CollectionStatistics::SniiScoringSegmentAccumulator segment_accumulator; + ASSERT_TRUE(stage_snii_fields_for_test(stats_.get(), {{L"1", 0, 0}, {L"2", 5, 10}}, + &segment_accumulator) + .ok()); + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, L"1", L"alpha", 0); + stats_->commit_snii_scoring_segment(std::move(segment_accumulator)); + + expect_collected_stats(L"1", 0, 0); + const float avg_dl = stats_->get_or_calculate_avg_dl(L"1"); + const float idf = stats_->get_or_calculate_idf(L"1", L"alpha"); + EXPECT_TRUE(std::isfinite(avg_dl)); + EXPECT_FLOAT_EQ(avg_dl, 0.0F); + EXPECT_TRUE(std::isfinite(idf)); + EXPECT_FLOAT_EQ(idf, static_cast(std::log(2.0))); +} + +// NULL ARRAY rows may keep tokens: they count in the document frequency but not in the indexed +// document count. idf and avgdl then use at least the document frequency and one document. +TEST_F(CollectionStatisticsTest, DocFrequencyAboveIndexedCountKeepsIdfPositive) { + CollectionStatistics::SniiScoringSegmentAccumulator segment_accumulator; + ASSERT_TRUE(stage_snii_fields_for_test(stats_.get(), {{L"1", 2, 6}, {L"2", 0, 4}}, + &segment_accumulator) + .ok()); + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, L"1", L"alpha", 3); + add_term_doc_frequency(&segment_accumulator.term_doc_freqs, L"2", L"beta", 1); + stats_->commit_snii_scoring_segment(std::move(segment_accumulator)); + + EXPECT_FLOAT_EQ(stats_->get_or_calculate_idf(L"1", L"alpha"), + static_cast(std::log(1 + (3 - 3 + 0.5) / (3 + 0.5)))); + EXPECT_GT(stats_->get_or_calculate_idf(L"1", L"alpha"), 0.0F); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(L"2"), 4.0F); + EXPECT_GT(stats_->get_or_calculate_idf(L"2", L"beta"), 0.0F); } TEST_F(CollectionStatisticsTest, LaterFieldFileNotFoundDoesNotPublishPartialSegment) { @@ -1269,11 +1341,11 @@ TEST_F(CollectionStatisticsDetailedTest, GetStatisticsWithValidData) { std::wstring field_name = L"test_field"; std::wstring term = L"test_term"; - stats_->set_total_num_docs(1000); + stats_->set_total_num_docs(field_name, 1000); stats_->set_total_num_tokens(field_name, 5000); stats_->set_term_doc_freq(field_name, term, 100); - EXPECT_EQ(stats_->get_doc_num(), 1000); + EXPECT_EQ(stats_->get_doc_num(field_name), 1000); EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), 5000); EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), 100); @@ -1289,7 +1361,7 @@ TEST_F(CollectionStatisticsDetailedTest, GetStatisticsThrowsWhenDataNotExists) { std::wstring nonexistent_term = L"nonexistent"; // Test exceptions for missing data - EXPECT_THROW(stats_->get_doc_num(), Exception); + EXPECT_THROW(stats_->get_doc_num(nonexistent_field), Exception); EXPECT_THROW(stats_->get_total_term_cnt_by_col(nonexistent_field), Exception); EXPECT_THROW(stats_->get_term_doc_freq_by_col(nonexistent_field, nonexistent_term), Exception); EXPECT_THROW(stats_->get_or_calculate_avg_dl(nonexistent_field), Exception); @@ -1300,14 +1372,14 @@ TEST_F(CollectionStatisticsDetailedTest, CachingMechanismWorks) { std::wstring field_name = L"test_field"; std::wstring term = L"test_term"; - stats_->set_total_num_docs(1000); + stats_->set_total_num_docs(field_name, 1000); stats_->set_total_num_tokens(field_name, 5000); stats_->set_term_doc_freq(field_name, term, 100); float first_avg_dl = stats_->get_or_calculate_avg_dl(field_name); float first_idf = stats_->get_or_calculate_idf(field_name, term); - stats_->set_total_num_docs(2000); + stats_->set_total_num_docs(field_name, 2000); stats_->set_total_num_tokens(field_name, 10000); stats_->set_term_doc_freq(field_name, term, 200); @@ -1322,16 +1394,24 @@ TEST_F(CollectionStatisticsDetailedTest, HandlesZeroValuesCorrectly) { std::wstring field_name = L"test_field"; std::wstring term = L"test_term"; - stats_->set_total_num_docs(0); - EXPECT_THROW(stats_->get_doc_num(), Exception); + EXPECT_THROW(stats_->get_doc_num(field_name), Exception); - stats_->set_total_num_docs(100); + stats_->set_total_num_docs(field_name, 100); stats_->set_total_num_tokens(field_name, 0); stats_->set_term_doc_freq(field_name, term, 0); EXPECT_EQ(stats_->get_total_term_cnt_by_col(field_name), 0); EXPECT_EQ(stats_->get_term_doc_freq_by_col(field_name, term), 0); EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(field_name), 0.0f); + + const std::wstring empty_field = L"empty_field"; + stats_->set_total_num_docs(empty_field, 0); + stats_->set_total_num_tokens(empty_field, 0); + stats_->set_term_doc_freq(empty_field, term, 0); + EXPECT_EQ(stats_->get_doc_num(empty_field), 0); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_avg_dl(empty_field), 0.0f); + EXPECT_FLOAT_EQ(stats_->get_or_calculate_idf(empty_field, term), + static_cast(std::log(2.0))); } TEST_F(CollectionStatisticsDetailedTest, IdfCalculationWithDifferentFrequencies) { @@ -1339,7 +1419,7 @@ TEST_F(CollectionStatisticsDetailedTest, IdfCalculationWithDifferentFrequencies) std::wstring common_term = L"common_term"; std::wstring rare_term = L"rare_term"; - stats_->set_total_num_docs(1000); + stats_->set_total_num_docs(field_name, 1000); stats_->set_term_doc_freq(field_name, common_term, 500); stats_->set_term_doc_freq(field_name, rare_term, 10); diff --git a/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp b/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp index 5cee6dcf1fea9e..195c54b0756b33 100644 --- a/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp +++ b/be/test/storage/index/snii/compaction/snii_index_compaction_test.cpp @@ -24,20 +24,26 @@ #include #include #include +#include #include #include #include #include +#include "common/config.h" #include "storage/index/snii/compaction/posting_run_merger.h" #include "storage/index/snii/format/norms_pod.h" #include "storage/index/snii/io/file_writer.h" +#include "storage/index/snii/query/bm25_scorer.h" #include "storage/index/snii/query/phrase_query.h" +#include "storage/index/snii/query/scoring_query.h" #include "storage/index/snii/query/term_query.h" #include "storage/index/snii/reader/logical_index_reader.h" #include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/stats/snii_stats_provider.h" #include "storage/index/snii/writer/snii_compound_writer.h" #include "storage/index/snii_query_test_util.h" +#include "util/defer_op.h" namespace { @@ -80,10 +86,15 @@ SniiIndexInput make_input(uint32_t doc_count, std::vector null_docids, // T2 input with norms (A2: analyzed indexes with positions always write norms). Caller-supplied // norms must match per-document posting frequencies so compaction can rebuild identical bytes. +// norms holds one entry per document that carries a norm: every non-NULL document plus the NULL +// documents in null_docids_with_norms (those with postings). SniiIndexInput make_norms_input(uint32_t doc_count, std::vector null_docids, - std::vector norms, std::vector terms) { + std::vector norms, std::vector terms, + std::vector null_docids_with_norms = {}) { SniiIndexInput input = make_input(doc_count, std::move(null_docids), std::move(terms)); input.encoded_norms = std::move(norms); + input.null_docids_with_norms = std::move(null_docids_with_norms); + input.write_norms = true; return input; } @@ -520,7 +531,8 @@ TEST(SniiIndexCompactionTest, NormsMergeMatchesRebuildAfterDeletesAndRemap) { /*doc_count=*/3, /*null_docids=*/ {2}, /*norms=*/ {2, 1, 1}, {make_term("alpha", {{.docid = 0, .positions = {0, 2}}, {.docid = 1, .positions = {0}}}), - make_term("beta", {{.docid = 2, .positions = {0}}})}), + make_term("beta", {{.docid = 2, .positions = {0}}})}, + /*null_docids_with_norms=*/ {2}), &source_zero, reader::LogicalIndexOpenMode::kCompaction); // Source 1: doc0 = alpha*1, doc1 = gamma*2; norms {1, 2}. build_index(make_norms_input( @@ -572,7 +584,8 @@ TEST(SniiIndexCompactionTest, NormsMergeMatchesRebuildAfterDeletesAndRemap) { make_term("gamma", {{.docid = 2, .positions = {0, 2}}})}), &rebuilt[0]); build_index(make_norms_input(/*doc_count=*/1, /*null_docids=*/ {0}, /*norms=*/ {1}, - {make_term("beta", {{.docid = 0, .positions = {0}}})}), + {make_term("beta", {{.docid = 0, .positions = {0}}})}, + /*null_docids_with_norms=*/ {0}), &rebuilt[1]); expect_identical_index_image(&merged_files[0], &rebuilt[0].file); expect_identical_index_image(&merged_files[1], &rebuilt[1].file); @@ -1017,4 +1030,211 @@ TEST(SniiIndexCompactionTest, StickyExecuteFailureAbortsNewDestinationSession) { EXPECT_FALSE(retry_file.finalized()); } +// One synthetic document: its NULL flag and how often each term occurs (0 = absent). A NULL +// document with occurrences models a nullable ARRAY row that kept its nested payload. +struct SyntheticDoc { + bool is_null = false; + uint32_t alpha = 0; + uint32_t beta = 0; +}; + +std::vector position_range(uint32_t first, uint32_t count) { + std::vector positions(count); + std::iota(positions.begin(), positions.end(), first); + return positions; +} + +// The input the column writer hands over for these documents: postings, NULL docids, the NULL +// docids that carry a norm, and the norms of every document that carries one. +SniiIndexInput make_synthetic_input(const std::vector& docs) { + std::vector alpha_docs; + std::vector beta_docs; + SniiIndexInput input = make_input(static_cast(docs.size()), {}, {}); + input.write_norms = true; + for (uint32_t docid = 0; docid < docs.size(); ++docid) { + const SyntheticDoc& doc = docs[docid]; + if (doc.alpha != 0) { + alpha_docs.push_back({.docid = docid, .positions = position_range(0, doc.alpha)}); + } + if (doc.beta != 0) { + beta_docs.push_back({.docid = docid, .positions = position_range(doc.alpha, doc.beta)}); + } + const uint32_t tokens = doc.alpha + doc.beta; + if (doc.is_null) { + input.null_docids.push_back(docid); + if (tokens == 0) { + continue; + } + input.null_docids_with_norms.push_back(docid); + } + input.encoded_norms.push_back(query::encode_norm(tokens)); + } + input.terms.push_back(make_term("alpha", std::move(alpha_docs))); + if (!beta_docs.empty()) { + input.terms.push_back(make_term("beta", std::move(beta_docs))); + } + return input; +} + +// Builds with enable_snii_sparse_norms off: the dense layout every earlier writer produced. +void build_index_with_dense_norms(SniiIndexInput input, OpenedIndex* out, + reader::LogicalIndexOpenMode open_mode) { + const bool saved_sparse_norms = doris::config::enable_snii_sparse_norms; + doris::Defer restore {[&] { doris::config::enable_snii_sparse_norms = saved_sparse_norms; }}; + doris::config::enable_snii_sparse_norms = false; + build_index(std::move(input), out, open_mode); +} + +bool index_has_sparse_norms(const reader::LogicalIndexReader& index) { + format::NormsPodReader norms; + assert_ok(index.open_norms(&norms)); + return norms.is_sparse(); +} + +std::vector score_all(const reader::LogicalIndexReader& index, + const std::string& term) { + stats::SniiStatsProvider stats; + assert_ok(stats::SniiStatsProvider::open(&index, &stats)); + std::vector docids; + assert_ok(term_query(index, term, &docids)); + roaring::Roaring candidates; + candidates.addMany(docids.size(), docids.data()); + std::vector scores; + assert_ok(query::scoring_query_candidates(index, stats, {{.physical_term = term, .idf = 2.5}}, + candidates, stats.avgdl(), query::Bm25Params {}, + &scores)); + return scores; +} + +// Direct compaction over a sparse-norms source, a dense-norms source written exactly as the +// earlier writers wrote it (with NULL rows) and a source without NULL rows. Each destination +// picks its own layout and is byte-identical to a fresh build of the merged documents; its BM25 +// scores equal those of a dense build of the same documents. With enable_snii_sparse_norms off +// the same merge writes dense destinations, byte-identical to a dense build. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST(SniiIndexCompactionTest, NormsMergeOfMixedLayoutsMatchesRebuild) { + std::vector sparse_docs(70000); + for (uint32_t docid = 0; docid < sparse_docs.size(); ++docid) { + SyntheticDoc& doc = sparse_docs[docid]; + doc.is_null = docid % 40 != 0; + if (!doc.is_null) { + doc.alpha = 1 + docid % 3; + doc.beta = docid % 80 == 0 ? 2 : 0; + } else if (docid % 4999 == 1) { + doc.alpha = 1; + } + } + std::vector dense_docs(66000); + for (uint32_t docid = 0; docid < dense_docs.size(); ++docid) { + SyntheticDoc& doc = dense_docs[docid]; + doc.is_null = docid % 25 != 0 && (docid < 1000 || docid >= 1500); + if (!doc.is_null && docid % 7 != 0) { + doc.alpha = 1 + docid % 2; + doc.beta = docid % 50 == 0 ? 1 : 0; + } + } + std::vector no_null_docs(300); + for (uint32_t docid = 0; docid < no_null_docs.size(); ++docid) { + no_null_docs[docid].alpha = docid % 2 == 0 ? 1 : 0; + no_null_docs[docid].beta = docid % 2 == 1 && docid != 299 ? 3 : 0; + } + + OpenedIndex sparse_source; + OpenedIndex dense_source; + OpenedIndex no_null_source; + build_index(make_synthetic_input(sparse_docs), &sparse_source, + reader::LogicalIndexOpenMode::kCompaction); + build_index_with_dense_norms(make_synthetic_input(dense_docs), &dense_source, + reader::LogicalIndexOpenMode::kCompaction); + build_index(make_synthetic_input(no_null_docs), &no_null_source, + reader::LogicalIndexOpenMode::kCompaction); + EXPECT_TRUE(index_has_sparse_norms(sparse_source.index)); + EXPECT_FALSE(index_has_sparse_norms(dense_source.index)); + EXPECT_FALSE(index_has_sparse_norms(no_null_source.index)); + + // Delete every 7th sparse-source document; fill destination 0 up to 65000 rows. + const std::vector*> sources = {&sparse_docs, &dense_docs, + &no_null_docs}; + RowIdConversionMap conversion(sources.size()); + std::array, 2> destination_docs; + for (size_t source = 0; source < sources.size(); ++source) { + for (uint32_t docid = 0; docid < sources[source]->size(); ++docid) { + if (source == 0 && docid % 7 == 3) { + conversion[source].push_back(kDeleted); + continue; + } + const uint32_t destination = destination_docs[0].size() < 65000 ? 0 : 1; + conversion[source].emplace_back( + destination, static_cast(destination_docs[destination].size())); + destination_docs[destination].push_back((*sources[source])[docid]); + } + } + const std::vector destination_rows = { + static_cast(destination_docs[0].size()), + static_cast(destination_docs[1].size())}; + ASSERT_EQ(destination_rows[0], 65000U); + auto validated = make_validated_conversion(&conversion, {70000, 66000, 300}, destination_rows); + ASSERT_NE(validated, nullptr); + compaction::SniiCompactionEligibility eligibility {.destination_writes_norms = true}; + + for (const bool sparse_norms : {true, false}) { + SCOPED_TRACE(std::string("enable_snii_sparse_norms=") + (sparse_norms ? "true" : "false")); + const bool saved_sparse_norms = doris::config::enable_snii_sparse_norms; + doris::Defer restore { + [&] { doris::config::enable_snii_sparse_norms = saved_sparse_norms; }}; + doris::config::enable_snii_sparse_norms = sparse_norms; + + std::unique_ptr plan; + assert_ok(SniiPlainT2MergePlan::prepare( + {&sparse_source.index, &dense_source.index, &no_null_source.index}, *validated, + eligibility, /*total_read_ahead_budget_bytes=*/1U << 20, + std::make_shared(nullptr, 64U << 20), &plan)); + + std::array merged_files; + std::array, 2> compounds; + std::array sessions = {nullptr, nullptr}; + for (size_t i = 0; i < compounds.size(); ++i) { + compounds[i] = std::make_unique(&merged_files[i]); + SniiIndexInput input = make_input(destination_rows[i], {}, {}); + input.config = plan->destination_index_config(); + input.write_norms = true; + assert_ok(compounds[i]->begin_streamed_index( + std::move(input), plan->take_destination_null_docids(i), &sessions[i])); + } + assert_ok(plan->execute(sessions)); + for (auto& compound : compounds) { + assert_ok(compound->finish()); + } + + for (size_t i = 0; i < merged_files.size(); ++i) { + SCOPED_TRACE("destination " + std::to_string(i)); + // A fresh build under the same config: sparse when on, dense when off. + OpenedIndex rebuilt; + build_index(make_synthetic_input(destination_docs[i]), &rebuilt); + expect_identical_index_image(&merged_files[i], &rebuilt.file); + + reader::SniiSegmentReader merged_segment; + reader::LogicalIndexReader merged_index; + assert_ok(reader::SniiSegmentReader::open(&merged_files[i], &merged_segment)); + assert_ok(merged_segment.open_index(kIndexId, kIndexSuffix, &merged_index)); + EXPECT_EQ(index_has_sparse_norms(merged_index), sparse_norms); + + OpenedIndex dense_rebuilt; + build_index_with_dense_norms(make_synthetic_input(destination_docs[i]), &dense_rebuilt, + reader::LogicalIndexOpenMode::kQuery); + EXPECT_FALSE(index_has_sparse_norms(dense_rebuilt.index)); + for (const char* term : {"alpha", "beta"}) { + const auto merged_scores = score_all(merged_index, term); + const auto dense_scores = score_all(dense_rebuilt.index, term); + ASSERT_FALSE(merged_scores.empty()); + ASSERT_EQ(merged_scores.size(), dense_scores.size()); + for (size_t j = 0; j < merged_scores.size(); ++j) { + ASSERT_EQ(merged_scores[j].docid, dense_scores[j].docid); + ASSERT_EQ(merged_scores[j].score, dense_scores[j].score); + } + } + } + } +} + } // namespace diff --git a/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp b/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp index eeb6d6e1f454b5..74a71eb51b8cb8 100644 --- a/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp +++ b/be/test/storage/index/snii/compaction/snii_streamed_session_test.cpp @@ -1055,7 +1055,10 @@ TEST(SniiStreamedWriterSessionTest, SessionOwnsMovedInputUntilContainerFinish) { {.docid = 2, .positions = {0}}}))); assert_ok(push_materialized(session, make_term("beta", {{.docid = 0, .positions = {1}}, {.docid = 2, .positions = {2}}}))); - assert_ok(session->set_encoded_norms(writer::TrackedEncodedNorms({7, 11, 13, 17}))); + // Only the non-NULL documents 0 and 2 carry a norm. + EXPECT_TRUE(session->set_encoded_norms(writer::TrackedEncodedNorms({7, 11, 13, 17})) + .is()); + assert_ok(session->set_encoded_norms(writer::TrackedEncodedNorms({7, 13}))); assert_ok(session->finish()); assert_ok(compound.finish()); @@ -1070,10 +1073,13 @@ TEST(SniiStreamedWriterSessionTest, SessionOwnsMovedInputUntilContainerFinish) { format::NormsPodReader norms; assert_ok(index.open_norms(&norms)); ASSERT_EQ(norms.doc_count(), 4U); + // Four documents are too few for the sparse layout to be smaller: the dense layout stores + // kEmptyDocumentNorm for the NULL documents. + ASSERT_FALSE(norms.is_sparse()); EXPECT_EQ(norms.encoded_norm(0), 7U); - EXPECT_EQ(norms.encoded_norm(1), 11U); + EXPECT_EQ(norms.encoded_norm(1), format::kEmptyDocumentNorm); EXPECT_EQ(norms.encoded_norm(2), 13U); - EXPECT_EQ(norms.encoded_norm(3), 17U); + EXPECT_EQ(norms.encoded_norm(3), format::kEmptyDocumentNorm); std::vector term_docs; assert_ok(term_query(index, "alpha", &term_docs)); @@ -1126,8 +1132,8 @@ TEST(SniiStreamedWriterSessionTest, ActiveAndFinishedSessionLifecycleIsEnforced) } // A2: A compaction destination declares write_norms before merging postings, then receives the -// rebuilt norms exactly once, with doc_count entries. Missing norms at finish poison the entire -// compound writer. +// rebuilt norms exactly once, with one entry per document that carries a norm. Missing norms at +// finish poison the entire compound writer. TEST(SniiStreamedWriterSessionTest, EncodedNormsAreLateBoundExactlyOnceBeforeFinish) { MemoryFile file; SniiCompoundWriter compound(&file); diff --git a/be/test/storage/index/snii/format/norms_pod_test.cpp b/be/test/storage/index/snii/format/norms_pod_test.cpp index 5639548edefd65..33986b78a75fce 100644 --- a/be/test/storage/index/snii/format/norms_pod_test.cpp +++ b/be/test/storage/index/snii/format/norms_pod_test.cpp @@ -19,19 +19,29 @@ #include +#include #include +#include +#include #include #include "common/status.h" #include "storage/index/snii/common/slice.h" #include "storage/index/snii/encoding/byte_sink.h" +#include "storage/index/snii/encoding/byte_source.h" +#include "storage/index/snii/encoding/crc32c.h" #include "storage/index/snii/encoding/section_framer.h" +#include "storage/index/snii/encoding/varint.h" #include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/query/bm25_scorer.h" using namespace doris::snii; using doris::Status; // RETURN_IF_ERROR expands to bare Status +using doris::snii::format::NormsLayout; using doris::snii::format::NormsPodReader; using doris::snii::format::NormsPodWriter; +using doris::snii::format::NormsSectionInput; +using doris::snii::format::NormsSectionPlan; namespace { @@ -217,3 +227,616 @@ TEST(SniiNormsPod, TryEncodedNormChecksBounds) { Status s = reader.try_encoded_norm(3, &v); EXPECT_TRUE(s.is()) << s.to_string(); } + +// --------------------------------------------------------------------------- +// Adaptive norms section: dense (kNormsPod) and sparse (kNormsSparse) layouts. +// --------------------------------------------------------------------------- +namespace { + +// One logical index's norms as every writer before the sparse layout stored them: a byte per +// document, kEmptyDocumentNorm for documents without a norm. +struct NormsCase { + uint32_t doc_count = 0; + std::vector nulls; + std::vector nulls_with_norms; + std::vector dense; + std::vector norms; // present-only view handed to the writer + + bool has_norm(uint32_t docid) const { + return !std::binary_search(nulls.begin(), nulls.end(), docid) || + std::binary_search(nulls_with_norms.begin(), nulls_with_norms.end(), docid); + } + + // Fills dense bytes for the normless documents and derives the present-only norms. + void finalize() { + norms.clear(); + for (uint32_t docid = 0; docid < doc_count; ++docid) { + if (has_norm(docid)) { + norms.push_back(dense[docid]); + } else { + dense[docid] = format::kEmptyDocumentNorm; + } + } + } + + NormsSectionInput input() const { + return {.doc_count = doc_count, + .null_docids = nulls, + .null_docids_with_norms = nulls_with_norms, + .norms = norms}; + } +}; + +std::vector write_section(const NormsCase& c, bool force_dense, NormsSectionPlan* plan) { + const Status status = format::plan_norms_section(c.input(), force_dense, plan); + EXPECT_TRUE(status.ok()) << status.to_string(); + ByteSink sink; + format::write_norms_section(c.input(), *plan, &sink); + EXPECT_EQ(sink.size(), plan->framed_bytes); + return sink.buffer(); +} + +// The dense section exactly as the pre-sparse writers framed it, assembled by hand: +// [u8 14][varint64 payload_len][varint64 doc_count][doc_count bytes][fixed32 crc32c]. +std::vector legacy_dense_section(const std::vector& dense) { + std::vector out; + out.push_back(14); + uint8_t varint[10]; + const size_t payload_len = varint_len(dense.size()) + dense.size(); + out.insert(out.end(), varint, varint + encode_varint64(payload_len, varint)); + out.insert(out.end(), varint, varint + encode_varint64(dense.size(), varint)); + out.insert(out.end(), dense.begin(), dense.end()); + const uint32_t crc = doris::snii::crc32c(Slice(out)); + for (int shift = 0; shift < 32; shift += 8) { + out.push_back(static_cast(crc >> shift)); + } + return out; +} + +void expect_lookups_match(const NormsCase& c, const NormsPodReader& reader) { + ASSERT_EQ(reader.doc_count(), c.doc_count); + // Collect mismatches instead of asserting per docid: this runs over millions of lookups. + size_t mismatches = 0; + std::string first_mismatch; + for (uint32_t docid = 0; docid < c.doc_count; ++docid) { + uint8_t norm = 0; + const Status status = reader.try_encoded_norm(docid, &norm); + bool matches = false; + if (c.has_norm(docid)) { + matches = status.ok() && norm == c.dense[docid] && + reader.encoded_norm(docid) == c.dense[docid]; + } else if (reader.is_sparse()) { + matches = status.is(); + } else { + matches = status.ok() && norm == format::kEmptyDocumentNorm; + } + if (!matches && mismatches++ == 0) { + first_mismatch = "docid=" + std::to_string(docid) + " norm=" + std::to_string(norm) + + " expected=" + std::to_string(c.dense[docid]) + " " + + status.to_string(); + } + } + ASSERT_EQ(mismatches, 0U) << first_mismatch; + uint8_t norm = 0; + EXPECT_TRUE( + reader.try_encoded_norm(c.doc_count, &norm).is()); +} + +struct SparseBlockHeader { + uint16_t key; + uint8_t kind; + uint32_t rank_base; + uint32_t payload_offset; +}; + +// Parses the documented sparse payload prefix: doc_count, present_count, bytes_per_norm and the +// block headers. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +std::vector sparse_block_headers(const std::vector& section, + uint8_t* bytes_per_norm) { + ByteSource src {Slice(section)}; + FramedSection framed; + EXPECT_TRUE(SectionFramer::read(src, &framed).ok()); + EXPECT_EQ(framed.type, static_cast(format::SectionType::kNormsSparse)); + ByteSource payload(framed.payload); + uint64_t value = 0; + EXPECT_TRUE(payload.get_varint64(&value).ok()); + EXPECT_TRUE(payload.get_varint64(&value).ok()); + EXPECT_TRUE(payload.get_u8(bytes_per_norm).ok()); + uint64_t block_count = 0; + EXPECT_TRUE(payload.get_varint64(&block_count).ok()); + std::vector headers; + for (uint64_t i = 0; i < block_count; ++i) { + SparseBlockHeader header {}; + uint8_t reserved = 0; + EXPECT_TRUE(payload.get_fixed16(&header.key).ok()); + EXPECT_TRUE(payload.get_u8(&header.kind).ok()); + EXPECT_TRUE(payload.get_u8(&reserved).ok()); + EXPECT_EQ(reserved, 0); + EXPECT_TRUE(payload.get_fixed32(&header.rank_base).ok()); + EXPECT_TRUE(payload.get_fixed32(&header.payload_offset).ok()); + headers.push_back(header); + } + return headers; +} + +NormsCase make_random_case(std::mt19937& rng, uint32_t doc_count, double null_ratio, + double run_bias, bool constant_norm) { + NormsCase c; + c.doc_count = doc_count; + c.dense.resize(doc_count); + std::uniform_int_distribution norm_dist(1, 255); + std::uniform_real_distribution unit(0.0, 1.0); + const auto constant = static_cast(norm_dist(rng)); + bool in_null = unit(rng) < null_ratio; + for (uint32_t docid = 0; docid < doc_count; ++docid) { + c.dense[docid] = constant_norm ? constant : static_cast(norm_dist(rng)); + // With run_bias close to 1 the null state rarely flips, producing long runs. + if (unit(rng) >= run_bias) { + in_null = unit(rng) < null_ratio; + } + if (in_null) { + c.nulls.push_back(docid); + if (unit(rng) < 0.01) { + c.nulls_with_norms.push_back(docid); + } + } + } + c.finalize(); + return c; +} + +} // namespace + +// Segments without NULL rows keep the dense section every earlier writer produced, byte for byte. +TEST(SniiNormsSection, NoNullsWritesLegacyDenseBytes) { + NormsCase c; + c.doc_count = 300; + for (uint32_t docid = 0; docid < c.doc_count; ++docid) { + c.dense.push_back(query::encode_norm(docid % 7)); + } + c.finalize(); + NormsSectionPlan plan; + const std::vector section = write_section(c, /*force_dense=*/false, &plan); + EXPECT_EQ(plan.layout, NormsLayout::kDense); + EXPECT_EQ(section, legacy_dense_section(c.dense)); + ByteSink legacy; + NormsPodWriter::finish(c.dense, &legacy); + EXPECT_EQ(section, legacy.buffer()); + EXPECT_EQ(section.size(), format::dense_norms_section_bytes(c.doc_count)); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(section), &reader).ok()); + EXPECT_FALSE(reader.is_sparse()); + EXPECT_EQ(reader.present_count(), c.doc_count); + expect_lookups_match(c, reader); +} + +// The dense layout of a segment with NULL rows stores kEmptyDocumentNorm for them, which is what +// the earlier writers stored (encode_norm(0)). NULL rows that kept tokens keep their norm. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST(SniiNormsSection, ForcedDenseMatchesLegacyBytesWithNulls) { + EXPECT_EQ(format::kEmptyDocumentNorm, query::encode_norm(0)); + NormsCase c; + c.doc_count = 20000; + c.dense.resize(c.doc_count); + for (uint32_t docid = 0; docid < c.doc_count; ++docid) { + c.dense[docid] = query::encode_norm(docid % 13); + if (docid % 10 != 0) { + c.nulls.push_back(docid); + } + } + c.nulls_with_norms = {1, 2, 19999}; + c.finalize(); + EXPECT_EQ(c.dense[1], query::encode_norm(1)); + EXPECT_EQ(c.dense[3], format::kEmptyDocumentNorm); + + NormsSectionPlan dense_plan; + const std::vector dense = write_section(c, /*force_dense=*/true, &dense_plan); + EXPECT_EQ(dense_plan.layout, NormsLayout::kDense); + EXPECT_EQ(dense, legacy_dense_section(c.dense)); + + NormsSectionPlan sparse_plan; + const std::vector sparse = write_section(c, /*force_dense=*/false, &sparse_plan); + ASSERT_EQ(sparse_plan.layout, NormsLayout::kSparse); + EXPECT_LT(sparse.size(), dense.size()); + + NormsPodReader dense_reader; + NormsPodReader sparse_reader; + ASSERT_TRUE(NormsPodReader::open(Slice(dense), &dense_reader).ok()); + ASSERT_TRUE(NormsPodReader::open(Slice(sparse), &sparse_reader).ok()); + EXPECT_FALSE(dense_reader.is_sparse()); + EXPECT_TRUE(sparse_reader.is_sparse()); + EXPECT_EQ(sparse_reader.present_count(), c.norms.size()); + expect_lookups_match(c, dense_reader); + expect_lookups_match(c, sparse_reader); +} + +TEST(SniiNormsSection, SparseRoundTripWithNormBytes) { + NormsCase c; + c.doc_count = 100000; + c.dense.resize(c.doc_count); + for (uint32_t docid = 0; docid < c.doc_count; ++docid) { + c.dense[docid] = query::encode_norm(docid % 200); + if (docid % 97 != 0) { + c.nulls.push_back(docid); + } + } + c.finalize(); + NormsSectionPlan plan; + const std::vector section = write_section(c, /*force_dense=*/false, &plan); + ASSERT_EQ(plan.layout, NormsLayout::kSparse); + EXPECT_EQ(plan.bytes_per_norm, 1); + uint8_t bytes_per_norm = 0; + const auto headers = sparse_block_headers(section, &bytes_per_norm); + EXPECT_EQ(bytes_per_norm, 1); + // Blocks 0 and 1; each holds a few hundred present docids -> ARRAY. + ASSERT_EQ(headers.size(), 2U); + EXPECT_EQ(headers[0].kind, 1); + EXPECT_EQ(headers[1].kind, 1); + EXPECT_EQ(headers[0].rank_base, 0U); + EXPECT_EQ(headers[1].rank_base, (65535U / 97) + 1); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(section), &reader).ok()); + ASSERT_TRUE(reader.is_sparse()); + EXPECT_EQ(reader.present_count(), c.norms.size()); + expect_lookups_match(c, reader); +} + +TEST(SniiNormsSection, SparseRoundTripWithConstantNorm) { + NormsCase c; + c.doc_count = 70000; + c.dense.assign(c.doc_count, query::encode_norm(3)); + for (uint32_t docid = 0; docid < c.doc_count; ++docid) { + if (docid < 1000 || docid >= 1100) { + c.nulls.push_back(docid); + } + } + c.finalize(); + NormsSectionPlan plan; + const std::vector section = write_section(c, /*force_dense=*/false, &plan); + ASSERT_EQ(plan.layout, NormsLayout::kSparse); + EXPECT_EQ(plan.bytes_per_norm, 0); + EXPECT_EQ(plan.constant_norm, query::encode_norm(3)); + uint8_t bytes_per_norm = 1; + const auto headers = sparse_block_headers(section, &bytes_per_norm); + EXPECT_EQ(bytes_per_norm, 0); + ASSERT_EQ(headers.size(), 1U); + EXPECT_EQ(headers[0].kind, 3); // one run + // [type][len][doc_count 3B][present 1B][bpn][block_count][12B header][data_len][2+6B][norm] + EXPECT_LT(section.size(), 40U); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(section), &reader).ok()); + EXPECT_TRUE(reader.is_sparse()); + EXPECT_EQ(reader.present_count(), 100U); + expect_lookups_match(c, reader); +} + +// Every block kind, a partial tail block and an empty middle block in one section. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST(SniiNormsSection, SparseBlockKinds) { + constexpr uint32_t kBlock = 1U << 16; + NormsCase c; + c.doc_count = 5 * kBlock + 1000; + c.dense.resize(c.doc_count); + std::mt19937 rng(7); + for (uint32_t docid = 0; docid < c.doc_count; ++docid) { + c.dense[docid] = static_cast(1 + docid % 251); + const uint32_t block = docid >> 16; + const uint32_t low = docid & 0xFFFF; + bool is_null = false; + switch (block) { + case 0: // ALL + is_null = false; + break; + case 1: // ARRAY: 100 present docids + is_null = low % 600 != 0 || low >= 60000; + break; + case 2: // RUNS: three long runs + is_null = !(low < 5000 || (low >= 20000 && low < 30000) || low >= 65000); + break; + case 3: // BITSET: scattered half + is_null = (rng() & 1U) != 0; + break; + case 4: // empty block + is_null = true; + break; + default: // partial tail, ALL except one null -> ARRAY or RUNS + is_null = low == 500; + break; + } + if (is_null) { + c.nulls.push_back(docid); + } + } + c.finalize(); + NormsSectionPlan plan; + const std::vector section = write_section(c, /*force_dense=*/false, &plan); + ASSERT_EQ(plan.layout, NormsLayout::kSparse); + uint8_t bytes_per_norm = 0; + const auto headers = sparse_block_headers(section, &bytes_per_norm); + ASSERT_EQ(headers.size(), 5U); + EXPECT_EQ(headers[0].key, 0); + EXPECT_EQ(headers[0].kind, 0); + EXPECT_EQ(headers[1].key, 1); + EXPECT_EQ(headers[1].kind, 1); + EXPECT_EQ(headers[2].key, 2); + EXPECT_EQ(headers[2].kind, 3); + EXPECT_EQ(headers[3].key, 3); + EXPECT_EQ(headers[3].kind, 2); + EXPECT_EQ(headers[4].key, 5); + EXPECT_EQ(headers[4].kind, 3); // two runs (14 bytes) beat 999 array entries + EXPECT_EQ(headers[1].rank_base, kBlock); + + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(section), &reader).ok()); + expect_lookups_match(c, reader); +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST(SniiNormsSection, LayoutChoice) { + // Tiny segments stay dense: the sparse header outweighs the saving. + NormsCase tiny; + tiny.doc_count = 4; + tiny.dense.assign(4, 2); + tiny.nulls = {1, 3}; + tiny.finalize(); + NormsSectionPlan plan; + write_section(tiny, false, &plan); + EXPECT_EQ(plan.layout, NormsLayout::kDense); + + // Every document NULL: a sparse section without blocks. + NormsCase all_null; + all_null.doc_count = 1000; + all_null.dense.assign(1000, 9); + for (uint32_t docid = 0; docid < 1000; ++docid) { + all_null.nulls.push_back(docid); + } + all_null.finalize(); + std::vector section = write_section(all_null, false, &plan); + ASSERT_EQ(plan.layout, NormsLayout::kSparse); + uint8_t bytes_per_norm = 1; + EXPECT_TRUE(sparse_block_headers(section, &bytes_per_norm).empty()); + EXPECT_EQ(bytes_per_norm, 0); + NormsPodReader reader; + ASSERT_TRUE(NormsPodReader::open(Slice(section), &reader).ok()); + EXPECT_EQ(reader.present_count(), 0U); + expect_lookups_match(all_null, reader); + write_section(all_null, /*force_dense=*/true, &plan); + EXPECT_EQ(plan.layout, NormsLayout::kDense); + + // NULL rows that all kept tokens leave no document without a norm: dense. + NormsCase all_with_norms; + all_with_norms.doc_count = 1000; + all_with_norms.dense.assign(1000, 9); + for (uint32_t docid = 0; docid < 1000; docid += 2) { + all_with_norms.nulls.push_back(docid); + } + all_with_norms.nulls_with_norms = all_with_norms.nulls; + all_with_norms.finalize(); + write_section(all_with_norms, false, &plan); + EXPECT_EQ(plan.layout, NormsLayout::kDense); + + // Scattered NULLs: one block of 6554 NULLs saves fewer bytes than its bitset costs: dense. + NormsCase scattered; + scattered.doc_count = 65536; + scattered.dense.resize(65536); + for (uint32_t docid = 0; docid < 65536; ++docid) { + scattered.dense[docid] = static_cast(1 + docid % 200); + if (docid % 10 == 0) { + scattered.nulls.push_back(docid); + } + } + scattered.finalize(); + write_section(scattered, false, &plan); + EXPECT_EQ(plan.layout, NormsLayout::kDense); + EXPECT_EQ(plan.framed_bytes, format::dense_norms_section_bytes(65536)); + + // Every third document NULL across two blocks: a bitset, a 2976-entry array and the norms + // beat a byte per document. + for (uint32_t docid = 0; docid < 65536; ++docid) { + scattered.dense[docid] = static_cast(1 + docid % 200); + } + scattered.doc_count = 70000; + scattered.dense.resize(70000, 7); + scattered.nulls.clear(); + for (uint32_t docid = 0; docid < 70000; docid += 3) { + scattered.nulls.push_back(docid); + } + scattered.finalize(); + section = write_section(scattered, false, &plan); + ASSERT_EQ(plan.layout, NormsLayout::kSparse); + const auto headers = sparse_block_headers(section, &bytes_per_norm); + ASSERT_EQ(headers.size(), 2U); + EXPECT_EQ(headers[0].kind, 2); + EXPECT_EQ(headers[1].kind, 1); + ASSERT_TRUE(NormsPodReader::open(Slice(section), &reader).ok()); + expect_lookups_match(scattered, reader); +} + +TEST(SniiNormsSection, PlanRejectsInvalidInput) { + NormsCase c; + c.doc_count = 10; + c.dense.assign(10, 1); + c.nulls = {2, 5}; + c.finalize(); + NormsSectionPlan plan; + + NormsSectionInput input = c.input(); + std::vector too_many(c.norms.size() + 1, 1); + input.norms = too_many; + EXPECT_TRUE(format::plan_norms_section(input, false, &plan) + .is()); + + std::vector not_a_subset = {3}; + std::vector nine(9, 1); + input = c.input(); + input.null_docids_with_norms = not_a_subset; + input.norms = nine; + EXPECT_TRUE(format::plan_norms_section(input, false, &plan) + .is()); + + std::vector unsorted = {5, 2}; + input = c.input(); + input.null_docids = unsorted; + EXPECT_TRUE(format::plan_norms_section(input, false, &plan) + .is()); + + std::vector outside = {2, 10}; + input = c.input(); + input.null_docids = outside; + EXPECT_TRUE(format::plan_norms_section(input, false, &plan) + .is()); +} + +// Randomized equivalence of sparse lookups against the dense oracle, and identical norms between +// the two layouts, across densities, run lengths and document counts that cross block edges. +TEST(SniiNormsSection, RandomizedLookupsMatchDenseOracle) { + std::mt19937 rng(20260917); + const std::vector doc_counts = {1, 2, 65535, 65536, 65537, 131072, 200003}; + const std::vector null_ratios = {0.0, 0.3, 0.9, 0.99, 0.999, 1.0}; + const std::vector run_biases = {0.0, 0.9, 0.9999}; + size_t sparse_cases = 0; + for (uint32_t doc_count : doc_counts) { + for (double null_ratio : null_ratios) { + for (double run_bias : run_biases) { + const bool constant = (rng() & 1U) != 0; + SCOPED_TRACE("doc_count=" + std::to_string(doc_count) + " null_ratio=" + + std::to_string(null_ratio) + " run_bias=" + std::to_string(run_bias) + + " constant=" + std::to_string(constant)); + const NormsCase c = + make_random_case(rng, doc_count, null_ratio, run_bias, constant); + NormsSectionPlan plan; + const std::vector section = write_section(c, false, &plan); + NormsSectionPlan dense_plan; + const std::vector dense = write_section(c, true, &dense_plan); + EXPECT_EQ(dense, legacy_dense_section(c.dense)); + EXPECT_LE(section.size(), dense.size()); + if (plan.layout == NormsLayout::kSparse) { + ++sparse_cases; + EXPECT_LT(section.size(), dense.size()); + } else { + EXPECT_EQ(section, dense); + } + NormsPodReader reader; + const Status status = NormsPodReader::open(Slice(section), &reader); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(reader.is_sparse(), plan.layout == NormsLayout::kSparse); + ASSERT_NO_FATAL_FAILURE(expect_lookups_match(c, reader)); + } + } + } + EXPECT_GT(sparse_cases, 20U); +} + +// A reader that predates the sparse layout accepts a norms region only when its length equals +// the dense length and its section type is kNormsPod (14). A sparse section fails both checks. +TEST(SniiNormsSection, SparseSectionFailsLegacyReaderChecks) { + NormsCase c; + c.doc_count = 50000; + c.dense.assign(c.doc_count, 4); + for (uint32_t docid = 0; docid < c.doc_count; docid += 1) { + if (docid % 1000 != 0) { + c.nulls.push_back(docid); + } + } + c.finalize(); + NormsSectionPlan plan; + const std::vector section = write_section(c, false, &plan); + ASSERT_EQ(plan.layout, NormsLayout::kSparse); + EXPECT_NE(section.size(), format::dense_norms_section_bytes(c.doc_count)); + EXPECT_EQ(section[0], static_cast(format::SectionType::kNormsSparse)); + EXPECT_NE(section[0], static_cast(format::SectionType::kNormsPod)); +} + +// Each structural field of a sparse payload is validated; a CRC-valid but inconsistent section +// is corruption, never a misread. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST(SniiNormsSection, RejectsCorruptSparsePayloads) { + constexpr auto kSparse = static_cast(format::SectionType::kNormsSparse); + auto open_payload = [](const ByteSink& payload) { + ByteSink framed; + SectionFramer::write(framed, kSparse, payload.view()); + NormsPodReader reader; + return NormsPodReader::open(framed.view(), &reader); + }; + auto header = [](ByteSink* out, uint16_t key, uint8_t kind, uint8_t reserved, uint32_t rank, + uint32_t offset) { + out->put_fixed16(key); + out->put_u8(kind); + out->put_u8(reserved); + out->put_fixed32(rank); + out->put_fixed32(offset); + }; + // A valid baseline: doc_count 10, present {2, 7}, ARRAY block, constant norm. + auto build = [&](uint64_t present, uint8_t bpn, uint8_t kind, uint8_t reserved, + std::vector lows, uint64_t data_len_delta, bool trailing) { + ByteSink payload; + payload.put_varint64(10); + payload.put_varint64(present); + payload.put_u8(bpn); + payload.put_varint64(1); + header(&payload, 0, kind, reserved, 0, 0); + payload.put_varint64(lows.size() * 2 + data_len_delta); + for (uint16_t low : lows) { + payload.put_fixed16(low); + } + for (uint64_t i = 0; i < data_len_delta; ++i) { + payload.put_u8(0); + } + payload.put_u8(5); + if (bpn == 1) { + payload.put_u8(6); + } + if (trailing) { + payload.put_u8(0); + } + return payload; + }; + EXPECT_TRUE(open_payload(build(2, 0, 1, 0, {2, 7}, 0, false)).ok()); + EXPECT_TRUE(open_payload(build(2, 1, 1, 0, {2, 7}, 0, false)).ok()); + auto corrupted = [&](const ByteSink& payload) { + return open_payload(payload).is(); + }; + EXPECT_TRUE(corrupted(build(2, 2, 1, 0, {2, 7}, 0, false))); // bytes_per_norm + EXPECT_TRUE(corrupted(build(2, 0, 4, 0, {2, 7}, 0, false))); // kind + EXPECT_TRUE(corrupted(build(2, 0, 1, 1, {2, 7}, 0, false))); // reserved + EXPECT_TRUE(corrupted(build(2, 0, 1, 0, {7, 2}, 0, false))); // unsorted lows + EXPECT_TRUE(corrupted(build(2, 0, 1, 0, {2, 10}, 0, false))); // low past span + EXPECT_TRUE(corrupted(build(3, 0, 1, 0, {2, 7}, 0, false))); // cardinality + EXPECT_TRUE(corrupted(build(2, 0, 1, 0, {2, 7}, 2, false))); // data length + EXPECT_TRUE(corrupted(build(2, 0, 1, 0, {2, 7}, 0, true))); // trailing byte + EXPECT_TRUE(corrupted(build(11, 0, 1, 0, {2, 7}, 0, false))); // present > doc_count + EXPECT_TRUE(corrupted(build(2, 0, 0, 0, {}, 0, false))); // ALL but 2 != span 10 + + // Block outside the document domain. + ByteSink payload; + payload.put_varint64(10); + payload.put_varint64(1); + payload.put_u8(0); + payload.put_varint64(1); + header(&payload, 1, 1, 0, 0, 0); + payload.put_varint64(2); + payload.put_fixed16(0); + payload.put_u8(5); + EXPECT_TRUE(corrupted(payload)); + + ByteSink runs; + runs.put_varint64(10); + runs.put_varint64(4); + runs.put_u8(0); + runs.put_varint64(1); + header(&runs, 0, 3, 0, 0, 0); + runs.put_varint64(2 + 12); + runs.put_fixed16(2); + runs.put_fixed16(1); // run [1, 2] + runs.put_fixed16(2); + runs.put_fixed16(0); + runs.put_fixed16(3); // adjacent run [3, 4]: runs must be separated + runs.put_fixed16(4); + runs.put_fixed16(2); + runs.put_u8(5); + EXPECT_TRUE(corrupted(runs)); +} diff --git a/be/test/storage/index/snii/format/null_bitmap_test.cpp b/be/test/storage/index/snii/format/null_bitmap_test.cpp index c3be8c51136e96..edd9732e63f9bd 100644 --- a/be/test/storage/index/snii/format/null_bitmap_test.cpp +++ b/be/test/storage/index/snii/format/null_bitmap_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "common/status.h" @@ -284,3 +285,141 @@ TEST(SniiNullBitmap, DecodedMemoryAccountsEachPortableContainer) { EXPECT_EQ(decoded_bytes, roaring_bytes + kContainerCount * kContainerMetadataBytes + kFixedBytes); } + +namespace { + +// Frames a bitmap exactly as the writers before run optimization did: addMany, then the portable +// serialization of whatever containers that produced. +std::vector LegacyUnoptimizedSection(const std::vector& nulls, + uint32_t doc_count) { + roaring::Roaring bitmap; + bitmap.addMany(nulls.size(), nulls.data()); + std::vector roaring_bytes(bitmap.getSizeInBytes()); + bitmap.write(roaring_bytes.data()); + ByteSink payload; + payload.put_varint64(doc_count); + payload.put_varint64(roaring_bytes.size()); + payload.put_bytes( + Slice(reinterpret_cast(roaring_bytes.data()), roaring_bytes.size())); + ByteSink sink; + SectionFramer::write(sink, kNullBitmapSectionType, payload.view()); + return sink.buffer(); +} + +uint16_t PortableCookie(const std::vector& framed) { + ByteSource src {Slice(framed)}; + FramedSection section; + EXPECT_TRUE(SectionFramer::read(src, §ion).ok()); + ByteSource payload(section.payload); + uint64_t value = 0; + EXPECT_TRUE(payload.get_varint64(&value).ok()); + EXPECT_TRUE(payload.get_varint64(&value).ok()); + uint16_t cookie = 0; + EXPECT_TRUE(payload.get_fixed16(&cookie).ok()); + return cookie; +} + +std::vector MostlyNull(uint32_t doc_count) { + std::vector nulls; + for (uint32_t docid = 0; docid < doc_count; ++docid) { + const bool present = (docid >= 1000 && docid < 1100) || + (docid >= 500000 && docid < 500010) || docid >= doc_count - 576; + if (!present) { + nulls.push_back(docid); + } + } + return nulls; +} + +} // namespace + +// A mostly NULL path over 16 containers: without run containers every one of them is an 8 KiB +// bitset; run-optimized they take a few bytes each, and the section still round-trips. +TEST(SniiNullBitmap, RunOptimizedMostlyNullSectionIsSmallAndRoundTrips) { + constexpr uint32_t kDocCount = 1U << 20; + const std::vector nulls = MostlyNull(kDocCount); + const std::vector legacy = LegacyUnoptimizedSection(nulls, kDocCount); + + NullBitmapWriter writer; + writer.add_many(nulls); + doris::snii::format::NullBitmapSerializationSizes sizes; + ASSERT_TRUE(writer.serialization_sizes(kDocCount, &sizes).ok()); + ByteSink sink; + ASSERT_TRUE(writer.finish(kDocCount, &sink).ok()); + const std::vector optimized = sink.buffer(); + EXPECT_EQ(optimized.size(), sizes.framed_bytes); + EXPECT_GT(legacy.size(), 16U * 8192); + EXPECT_LT(optimized.size() * 100, legacy.size()); + EXPECT_EQ(PortableCookie(optimized), 12347); // SERIAL_COOKIE: run containers present + EXPECT_EQ(PortableCookie(legacy), 12346); // SERIAL_COOKIE_NO_RUNCONTAINER + + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(optimized), &reader).ok()); + EXPECT_EQ(reader.doc_count(), kDocCount); + EXPECT_EQ(reader.null_count(), nulls.size()); + std::vector decoded; + reader.append_docids(decoded); + EXPECT_EQ(decoded, nulls); + uint64_t decoded_bytes = 0; + ASSERT_TRUE(NullBitmapReader::decoded_memory_bytes(Slice(optimized), &decoded_bytes).ok()); + EXPECT_LT(decoded_bytes, 16U * 8192); +} + +// Sections written before run optimization (array and bitset containers only) stay readable. +TEST(SniiNullBitmap, ReadsLegacyUnoptimizedSections) { + const std::vector>> cases = { + {1U << 20, MostlyNull(1U << 20)}, + {5000, {0, 3, 7, 11, 100, 4000}}, + {200000, + [] { + std::vector nulls; + for (uint32_t docid = 0; docid < 200000; ++docid) { + if (docid % 5 != 0) { + nulls.push_back(docid); + } + } + return nulls; + }()}, + }; + for (const auto& [doc_count, nulls] : cases) { + const std::vector legacy = LegacyUnoptimizedSection(nulls, doc_count); + ASSERT_EQ(PortableCookie(legacy), 12346); + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(Slice(legacy), &reader).ok()); + EXPECT_EQ(reader.doc_count(), doc_count); + EXPECT_EQ(reader.null_count(), nulls.size()); + std::vector decoded; + reader.append_docids(decoded); + EXPECT_EQ(decoded, nulls); + roaring::Roaring copy; + reader.copy_to(©); + EXPECT_EQ(copy.cardinality(), nulls.size()); + uint64_t decoded_bytes = 0; + EXPECT_TRUE(NullBitmapReader::decoded_memory_bytes(Slice(legacy), &decoded_bytes).ok()); + } +} + +// Sizes are computed on the optimized bitmap, also after more docids arrive. +TEST(SniiNullBitmap, SerializationSizesFollowLaterAdds) { + NullBitmapWriter writer; + std::vector run(70000); + for (uint32_t i = 0; i < run.size(); ++i) { + run[i] = i; + } + writer.add_many(run); + doris::snii::format::NullBitmapSerializationSizes first; + ASSERT_TRUE(writer.serialization_sizes(100000, &first).ok()); + writer.add_null(90000); + doris::snii::format::NullBitmapSerializationSizes second; + ASSERT_TRUE(writer.serialization_sizes(100000, &second).ok()); + EXPECT_GT(second.roaring_bytes, first.roaring_bytes); + ByteSink sink; + ASSERT_TRUE(writer.finish(100000, &sink).ok()); + EXPECT_EQ(sink.size(), second.framed_bytes); + NullBitmapReader reader; + ASSERT_TRUE(NullBitmapReader::open(sink.view(), &reader).ok()); + EXPECT_EQ(reader.null_count(), 70001U); + EXPECT_TRUE(reader.is_null(69999)); + EXPECT_FALSE(reader.is_null(70000)); + EXPECT_TRUE(reader.is_null(90000)); +} diff --git a/be/test/storage/index/snii/snii_sparse_norms_test.cpp b/be/test/storage/index/snii/snii_sparse_norms_test.cpp new file mode 100644 index 00000000000000..0f32f0dbd4b803 --- /dev/null +++ b/be/test/storage/index/snii/snii_sparse_norms_test.cpp @@ -0,0 +1,437 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// End-to-end sparse norms through the production column writer: with the BE config +// enable_snii_sparse_norms on, NULL-heavy scalar and ARRAY columns get the sparse norms section; +// with it off, the same input produces the section every earlier writer produced; BM25 scores are +// identical between the two layouts. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/check.h" +#include "common/config.h" +#include "common/status.h" +#include "io/fs/local_file_system.h" +#include "storage/index/index_file_reader.h" +#include "storage/index/index_file_writer.h" +#include "storage/index/inverted/inverted_index_desc.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/format/norms_pod.h" +#include "storage/index/snii/query/bm25_scorer.h" +#include "storage/index/snii/query/scoring_query.h" +#include "storage/index/snii/query/term_query.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/snii_index_writer.h" +#include "storage/index/snii/stats/snii_stats_provider.h" +#include "storage/tablet/tablet_schema.h" +#include "util/defer_op.h" +#include "util/slice.h" + +namespace doris::segment_v2 { +namespace { + +constexpr int64_t kIndexId = 7101; + +using ScalarRow = std::optional; +using ArrayRow = std::optional>>; + +TabletIndex make_meta() { + TabletIndexPB pb; + pb.set_index_type(IndexType::INVERTED); + pb.set_index_id(kIndexId); + pb.set_index_name("sparse_norms"); + pb.add_col_unique_id(0); + pb.mutable_properties()->insert({"parser", "english"}); + pb.mutable_properties()->insert({"support_phrase", "true"}); + TabletIndex meta; + meta.init_from_pb(pb); + return meta; +} + +// Runs `write` with enable_snii_sparse_norms set to `enabled`, then restores the config. +Status with_sparse_norms_config(bool enabled, const std::function& write) { + const bool saved_sparse_norms = config::enable_snii_sparse_norms; + Defer restore {[&] { config::enable_snii_sparse_norms = saved_sparse_norms; }}; + config::enable_snii_sparse_norms = enabled; + return write(); +} + +Status open_writer(const std::string& prefix, std::unique_ptr* out) { + const std::string file_path = InvertedIndexDescriptor::get_index_file_path_v2(prefix); + auto fs = io::global_local_filesystem(); + bool exists = false; + RETURN_IF_ERROR(fs->exists(file_path, &exists)); + if (exists) { + RETURN_IF_ERROR(fs->delete_file(file_path)); + } + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(fs->create_file(file_path, &file_writer)); + *out = std::make_unique(fs, prefix, "sparse_norms_rowset", /*seg_id=*/0, + InvertedIndexStorageFormatPB::SNII, + std::move(file_writer), + /*can_use_ram_dir=*/true, /*tablet_id=*/901); + return Status::OK(); +} + +// Mirrors ScalarColumnWriter::append_nullable: runs of non-NULL rows go to add_values, NULL runs +// to add_nulls. +Status write_scalar(const std::string& prefix, const TabletIndex& meta, + const std::vector& rows) { + std::unique_ptr file_writer; + RETURN_IF_ERROR(open_writer(prefix, &file_writer)); + SniiIndexColumnWriter writer(file_writer.get(), &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + RETURN_IF_ERROR(writer.init()); + std::vector batch; + uint32_t null_run = 0; + for (const auto& row : rows) { + if (row.has_value()) { + if (null_run != 0) { + RETURN_IF_ERROR(writer.add_nulls(null_run)); + null_run = 0; + } + batch.emplace_back(*row); + } else { + if (!batch.empty()) { + RETURN_IF_ERROR(writer.add_values("content", batch.data(), batch.size())); + batch.clear(); + } + ++null_run; + } + } + if (null_run != 0) { + RETURN_IF_ERROR(writer.add_nulls(null_run)); + } + if (!batch.empty()) { + RETURN_IF_ERROR(writer.add_values("content", batch.data(), batch.size())); + } + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(file_writer->begin_close()); + return file_writer->finish_close(); +} + +// Mirrors ArrayColumnWriter::append_nullable: every row goes to add_array_values (a NULL row may +// keep a nested payload), then add_array_nulls marks the NULL rows. Written in two batches. +Status write_array(const std::string& prefix, const TabletIndex& meta, + const std::vector& rows, const std::vector& row_is_null) { + std::unique_ptr file_writer; + RETURN_IF_ERROR(open_writer(prefix, &file_writer)); + SniiIndexColumnWriter writer(file_writer.get(), &meta, FieldType::OLAP_FIELD_TYPE_VARCHAR); + RETURN_IF_ERROR(writer.init()); + const size_t split = rows.size() / 2; + for (const auto& [begin, end] : + {std::pair {0, split}, std::pair {split, rows.size()}}) { + std::vector storage; + std::vector element_nulls; + std::vector offsets {0}; + std::vector null_map; + for (size_t i = begin; i < end; ++i) { + if (rows[i].has_value()) { + for (const auto& element : *rows[i]) { + storage.push_back(element.value_or("")); + element_nulls.push_back(element.has_value() ? 0 : 1); + } + } + offsets.push_back(storage.size()); + null_map.push_back(row_is_null[i] ? 1 : 0); + } + std::vector elements; + elements.reserve(storage.size()); + for (const auto& value : storage) { + elements.emplace_back(value); + } + RETURN_IF_ERROR(writer.add_array_values( + sizeof(Slice), elements.data(), element_nulls.data(), + reinterpret_cast(offsets.data()), end - begin)); + RETURN_IF_ERROR(writer.add_array_nulls(null_map.data(), end - begin)); + } + RETURN_IF_ERROR(writer.finish()); + RETURN_IF_ERROR(file_writer->begin_close()); + return file_writer->finish_close(); +} + +struct OpenedIndex { + std::shared_ptr file_reader; + std::unique_ptr index; + snii::stats::SniiStatsProvider stats; + snii::format::NormsPodReader norms; + std::vector norms_section; +}; + +Status open_index(const std::string& prefix, const TabletIndex& meta, OpenedIndex* out) { + out->file_reader = std::make_shared(io::global_local_filesystem(), prefix, + InvertedIndexStorageFormatPB::SNII); + RETURN_IF_ERROR(out->file_reader->init()); + auto index = out->file_reader->open_snii_index(&meta); + if (!index.has_value()) { + return index.error(); + } + out->index = std::move(index.value()); + RETURN_IF_ERROR(snii::stats::SniiStatsProvider::open(out->index.get(), &out->stats)); + RETURN_IF_ERROR(out->index->open_norms(&out->norms)); + + std::ifstream file(InvertedIndexDescriptor::get_index_file_path_v2(prefix), std::ios::binary); + const std::vector bytes((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + const auto& region = out->index->section_refs().norms; + DORIS_CHECK_LE(region.offset + region.length, bytes.size()); + out->norms_section.assign( + bytes.begin() + static_cast(region.offset), + bytes.begin() + static_cast(region.offset + region.length)); + return Status::OK(); +} + +// Scores every posting of `term` with fixed collection statistics. +Status score_term(const OpenedIndex& opened, const std::string& term, + std::vector* out) { + std::vector docids; + RETURN_IF_ERROR(snii::query::term_query(*opened.index, term, &docids)); + roaring::Roaring candidates; + candidates.addMany(docids.size(), docids.data()); + return snii::query::scoring_query_candidates( + *opened.index, opened.stats, {{.physical_term = term, .idf = 1.7}}, candidates, + opened.stats.avgdl(), snii::query::Bm25Params {}, out); +} + +void expect_same_scores(const OpenedIndex& sparse, const OpenedIndex& dense, + const std::string& term, size_t expected_hits) { + std::vector sparse_scores; + std::vector dense_scores; + Status status = score_term(sparse, term, &sparse_scores); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = score_term(dense, term, &dense_scores); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_EQ(sparse_scores.size(), expected_hits); + ASSERT_EQ(sparse_scores.size(), dense_scores.size()); + for (size_t i = 0; i < sparse_scores.size(); ++i) { + EXPECT_EQ(sparse_scores[i].docid, dense_scores[i].docid); + // Bit-identical: both layouts hand the scorer the same norm byte. + EXPECT_EQ(sparse_scores[i].score, dense_scores[i].score) + << "docid " << sparse_scores[i].docid; + EXPECT_GT(sparse_scores[i].score, 0.0); + } +} + +class SniiSparseNormsTest : public ::testing::Test { +protected: + void SetUp() override { + _dir = "./ut_dir/snii_sparse_norms_test"; + static_cast(io::global_local_filesystem()->delete_directory(_dir)); + ASSERT_TRUE(io::global_local_filesystem()->create_directory(_dir).ok()); + } + void TearDown() override { + static_cast(io::global_local_filesystem()->delete_directory(_dir)); + } + + std::string _dir; +}; + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST_F(SniiSparseNormsTest, NullHeavyScalarColumnWritesSparseNorms) { + constexpr uint32_t kRows = 150000; + std::vector rows(kRows); + std::vector token_counts(kRows, 0); + size_t present = 0; + for (uint32_t row = 0; row < kRows; ++row) { + // One present row in 100, plus a present stretch; one empty string (present, no token). + if (row % 100 == 0 || (row >= 70000 && row < 70200) || row == 12345) { + std::string value = row == 12345 ? "" : "alpha"; + for (uint32_t i = 0; i < row % 4; ++i) { + value += " beta"; + } + if (row == 12345) { + value.clear(); + } + token_counts[row] = row == 12345 ? 0 : 1 + row % 4; + rows[row] = std::move(value); + ++present; + } + } + // The sparse layout is on by default. + EXPECT_TRUE(config::enable_snii_sparse_norms); + const TabletIndex meta = make_meta(); + const std::string sparse_prefix = _dir + "/scalar_sparse"; + const std::string dense_prefix = _dir + "/scalar_dense"; + // The same rows written with the config on and off. + ASSERT_TRUE(with_sparse_norms_config(true, [&] { + return write_scalar(sparse_prefix, meta, rows); + }).ok()); + ASSERT_TRUE(with_sparse_norms_config(false, [&] { + return write_scalar(dense_prefix, meta, rows); + }).ok()); + + OpenedIndex sparse; + OpenedIndex dense; + Status status = open_index(sparse_prefix, meta, &sparse); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = open_index(dense_prefix, meta, &dense); + ASSERT_TRUE(status.ok()) << status.to_string(); + + EXPECT_EQ(sparse.index->stats().doc_count, kRows); + EXPECT_EQ(sparse.index->stats().indexed_doc_count, present); + EXPECT_EQ(dense.index->stats().indexed_doc_count, present); + ASSERT_TRUE(sparse.norms.is_sparse()); + ASSERT_FALSE(dense.norms.is_sparse()); + EXPECT_EQ(sparse.norms.present_count(), present); + EXPECT_EQ(sparse.norms_section[0], + static_cast(snii::format::SectionType::kNormsSparse)); + EXPECT_EQ(dense.norms_section[0], static_cast(snii::format::SectionType::kNormsPod)); + EXPECT_EQ(dense.norms_section.size(), snii::format::dense_norms_section_bytes(kRows)); + EXPECT_LT(sparse.norms_section.size() * 20, dense.norms_section.size()); + + // The dense section is the one earlier writers produced: one byte per row, encode_norm(0) + // for NULL rows. + std::vector legacy(kRows); + for (uint32_t row = 0; row < kRows; ++row) { + legacy[row] = snii::query::encode_norm(token_counts[row]); + } + snii::ByteSink legacy_section; + snii::format::NormsPodWriter::finish(legacy, &legacy_section); + EXPECT_EQ(dense.norms_section, legacy_section.buffer()); + + for (uint32_t row = 0; row < kRows; ++row) { + uint8_t sparse_norm = 0; + const Status sparse_status = sparse.stats.encoded_norm(row, &sparse_norm); + if (rows[row].has_value()) { + ASSERT_TRUE(sparse_status.ok()) << row << " " << sparse_status.to_string(); + ASSERT_EQ(sparse_norm, legacy[row]) << row; + } else { + ASSERT_TRUE(sparse_status.is()) << row; + } + uint8_t dense_norm = 0; + ASSERT_TRUE(dense.stats.encoded_norm(row, &dense_norm).ok()); + ASSERT_EQ(dense_norm, legacy[row]) << row; + } + EXPECT_DOUBLE_EQ(sparse.stats.avgdl(), dense.stats.avgdl()); + expect_same_scores(sparse, dense, "alpha", present - 1); + // Multiples of 100 are multiples of 4 and carry no "beta"; the stretch adds 150 rows with one. + expect_same_scores(sparse, dense, "beta", 150); +} + +// A NULL ARRAY row that kept its nested payload has postings, so it keeps a norm; NULL rows +// without tokens, empty arrays and arrays of NULL elements behave as documented. +// NOLINTNEXTLINE(readability-function-cognitive-complexity) -- GTest assertions inflate it. +TEST_F(SniiSparseNormsTest, NullableArrayRowsKeepNormsOnlyWithTokens) { + constexpr uint32_t kRows = 80000; + std::vector rows(kRows); + std::vector row_is_null(kRows, true); + std::vector expected_null_docids_with_norms; + for (uint32_t row = 0; row < kRows; ++row) { + if (row % 50 == 0) { + // Non-NULL rows: tokens across elements, an empty array, an array of NULL elements. + row_is_null[row] = false; + if (row % 150 == 0) { + rows[row] = std::vector> {}; + } else if (row % 250 == 0) { + rows[row] = std::vector> {std::nullopt, std::nullopt}; + } else { + rows[row] = std::vector> {"alpha beta", std::nullopt, + "alpha"}; + } + } else if (row % 997 == 0) { + // NULL row keeping a nested payload. + rows[row] = std::vector> {"alpha gamma"}; + expected_null_docids_with_norms.push_back(row); + } + } + const TabletIndex meta = make_meta(); + const std::string sparse_prefix = _dir + "/array_sparse"; + const std::string dense_prefix = _dir + "/array_dense"; + ASSERT_TRUE(with_sparse_norms_config(true, [&] { + return write_array(sparse_prefix, meta, rows, row_is_null); + }).ok()); + ASSERT_TRUE(with_sparse_norms_config(false, [&] { + return write_array(dense_prefix, meta, rows, row_is_null); + }).ok()); + + OpenedIndex sparse; + OpenedIndex dense; + Status status = open_index(sparse_prefix, meta, &sparse); + ASSERT_TRUE(status.ok()) << status.to_string(); + status = open_index(dense_prefix, meta, &dense); + ASSERT_TRUE(status.ok()) << status.to_string(); + ASSERT_TRUE(sparse.norms.is_sparse()); + ASSERT_FALSE(dense.norms.is_sparse()); + const uint32_t non_null_rows = kRows / 50; + EXPECT_EQ(sparse.index->stats().indexed_doc_count, non_null_rows); + EXPECT_EQ(sparse.norms.present_count(), non_null_rows + expected_null_docids_with_norms.size()); + + for (uint32_t row = 0; row < kRows; ++row) { + uint8_t sparse_norm = 0; + uint8_t dense_norm = 0; + ASSERT_TRUE(dense.stats.encoded_norm(row, &dense_norm).ok()); + const Status sparse_status = sparse.stats.encoded_norm(row, &sparse_norm); + const bool has_tokens = + rows[row].has_value() && row % 150 != 0 && !(row % 50 == 0 && row % 250 == 0); + if (!row_is_null[row] || has_tokens) { + ASSERT_TRUE(sparse_status.ok()) << row << " " << sparse_status.to_string(); + ASSERT_EQ(sparse_norm, dense_norm) << row; + uint64_t tokens = 0; + if (has_tokens) { + tokens = row_is_null[row] ? 2 : 3; + } + ASSERT_EQ(dense_norm, snii::query::encode_norm(tokens)) << row; + } else { + ASSERT_TRUE(sparse_status.is()) << row; + ASSERT_EQ(dense_norm, snii::format::kEmptyDocumentNorm) << row; + } + } + // "alpha" postings include the NULL rows that kept tokens; scoring reads their norms. + size_t alpha_rows = expected_null_docids_with_norms.size(); + for (uint32_t row = 0; row < kRows; row += 50) { + alpha_rows += (row % 150 != 0 && row % 250 != 0) ? 1 : 0; + } + expect_same_scores(sparse, dense, "alpha", alpha_rows); + expect_same_scores(sparse, dense, "gamma", expected_null_docids_with_norms.size()); +} + +// Without NULL rows every document carries a norm, so the writer keeps the dense layout whatever +// enable_snii_sparse_norms says, and the two files are identical. +TEST_F(SniiSparseNormsTest, SegmentWithoutNullsIsIdenticalInBothModes) { + std::vector rows; + for (uint32_t row = 0; row < 5000; ++row) { + rows.emplace_back(row % 2 == 0 ? "alpha beta" : "gamma"); + } + const TabletIndex meta = make_meta(); + const std::string adaptive_prefix = _dir + "/no_null_adaptive"; + const std::string dense_prefix = _dir + "/no_null_dense"; + ASSERT_TRUE(with_sparse_norms_config(true, [&] { + return write_scalar(adaptive_prefix, meta, rows); + }).ok()); + ASSERT_TRUE(with_sparse_norms_config(false, [&] { + return write_scalar(dense_prefix, meta, rows); + }).ok()); + auto read_file = [](const std::string& prefix) { + std::ifstream file(InvertedIndexDescriptor::get_index_file_path_v2(prefix), + std::ios::binary); + return std::vector((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + }; + EXPECT_EQ(read_file(adaptive_prefix), read_file(dense_prefix)); +} + +} // namespace +} // namespace doris::segment_v2 diff --git a/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp b/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp index 1fd58df5042565..732569ea20971c 100644 --- a/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp +++ b/be/test/storage/index/snii/writer/snii_compound_writer_test.cpp @@ -559,7 +559,8 @@ SniiIndexInput MakeIndexWithAllAuxiliarySections(MemoryReporter* reporter) { in.config = IndexConfig::kDocsPositions; in.doc_count = 3; in.null_docids = {2}; - in.encoded_norms = {1, 2, 3}; + // One norm for each non-NULL document. + in.encoded_norms = {1, 2}; in.terms.push_back(MakeTerm("apple", {0, 1}, true)); in.mem_reporter = reporter; return in; diff --git a/be/test/storage/index/snii/writer/snii_shared_null_bitmap_test.cpp b/be/test/storage/index/snii/writer/snii_shared_null_bitmap_test.cpp new file mode 100644 index 00000000000000..f9cdc1750ef8cb --- /dev/null +++ b/be/test/storage/index/snii/writer/snii_shared_null_bitmap_test.cpp @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A VARIANT column copies every index definition to each materialized subcolumn, +// so one container holds one logical index per (definition, subcolumn), and the +// definitions on one subcolumn share its suffix and its NULL rows. The compound +// writer stores their identical null bitmap once and points every such index at +// that region. These tests pin down when that happens, that every index still +// reads its own NULL rows back through the load, compaction and rewrite paths, +// and how many bytes it saves. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "storage/index/snii/format/core_metadata.h" +#include "storage/index/snii/format/format_constants.h" +#include "storage/index/snii/reader/logical_index_reader.h" +#include "storage/index/snii/reader/snii_segment_reader.h" +#include "storage/index/snii/writer/logical_index_writer.h" +#include "storage/index/snii/writer/memory_reporter.h" +#include "storage/index/snii/writer/snii_compound_writer.h" +#include "storage/index/snii_query_test_util.h" + +namespace { + +using namespace doris::snii; // NOLINT +using namespace doris::snii::snii_test; // NOLINT +using doris::Status; +using format::RegionRef; +using format::SectionRefs; +using writer::SniiCompoundWriter; +using writer::SniiIndexInput; +using writer::SniiStreamedIndexSession; + +// The escaped suffix shape a VARIANT subcolumn index carries (v.payload.title). +const std::string kPath = "v%2Epayload%2Etitle"; +const std::string kOtherPath = "v%2Epayload%2Ebody"; + +// A text index on `suffix` whose NULL rows are `null_docids`; the first non-NULL +// document carries the term "x". +SniiIndexInput PathIndex(uint64_t index_id, const std::string& suffix, uint32_t doc_count, + std::vector null_docids) { + SniiIndexInput input; + input.index_id = index_id; + input.index_suffix = suffix; + input.config = format::IndexConfig::kDocsPositions; + input.doc_count = doc_count; + uint32_t first_present = 0; + while (std::ranges::binary_search(null_docids, first_present)) { + ++first_present; + } + input.terms = {make_term("x", {{.docid = first_present, .positions = {0}}})}; + input.null_docids = std::move(null_docids); + return input; +} + +MemoryFile WriteContainer(const std::vector& inputs) { + MemoryFile file; + SniiCompoundWriter compound(&file); + for (const SniiIndexInput& input : inputs) { + EXPECT_TRUE(compound.add_logical_index(input).ok()); + } + EXPECT_TRUE(compound.finish().ok()); + return file; +} + +RegionRef NullRegion(MemoryFile* file, uint64_t index_id, const std::string& suffix) { + reader::SniiSegmentReader segment; + EXPECT_TRUE(reader::SniiSegmentReader::open(file, &segment).ok()); + SectionRefs refs; + EXPECT_TRUE(segment.section_refs_for_index(index_id, suffix, &refs).ok()); + return refs.null_bitmap; +} + +// Reads one index's NULL rows through the same path compaction uses, which also +// checks the bitmap's doc and null counts against that index's own stats. +std::vector ReadNulls(MemoryFile* file, uint64_t index_id, const std::string& suffix) { + reader::SniiSegmentReader segment; + EXPECT_TRUE(reader::SniiSegmentReader::open(file, &segment).ok()); + reader::LogicalIndexReader index; + EXPECT_TRUE(segment.open_index(index_id, suffix, &index).ok()); + std::vector nulls; + EXPECT_TRUE(index.read_null_docids(&nulls).ok()); + EXPECT_EQ(index.stats().null_count, nulls.size()); + return nulls; +} + +size_t CountOccurrences(const std::vector& haystack, const std::vector& needle) { + size_t count = 0; + for (auto it = haystack.begin();; ++it) { + it = std::search(it, haystack.end(), needle.begin(), needle.end()); + if (it == haystack.end()) { + return count; + } + ++count; + } +} + +std::vector Bytes(const MemoryFile& file, const RegionRef& region) { + const auto begin = file.data().begin() + static_cast(region.offset); + return {begin, begin + static_cast(region.length)}; +} + +} // namespace + +TEST(SniiSharedNullBitmap, DefinitionsOnOneSubcolumnReferenceOneRegion) { + const std::vector nulls = {1, 3, 5, 6}; + MemoryFile file = + WriteContainer({PathIndex(11, kPath, 8, nulls), PathIndex(12, kPath, 8, nulls), + PathIndex(13, kPath, 8, nulls)}); + + const RegionRef first = NullRegion(&file, 11, kPath); + ASSERT_GT(first.length, 0U); + for (uint64_t index_id : {12, 13}) { + const RegionRef region = NullRegion(&file, index_id, kPath); + EXPECT_EQ(region.offset, first.offset) << "index " << index_id; + EXPECT_EQ(region.length, first.length) << "index " << index_id; + } + EXPECT_EQ(CountOccurrences(file.data(), Bytes(file, first)), 1U) + << "the bitmap bytes must be stored once"; + for (uint64_t index_id : {11, 12, 13}) { + EXPECT_EQ(ReadNulls(&file, index_id, kPath), nulls) << "index " << index_id; + } +} + +TEST(SniiSharedNullBitmap, DifferentBitmapsOnOneSubcolumnAreNotShared) { + // Different NULL rows. + { + MemoryFile file = + WriteContainer({PathIndex(11, kPath, 8, {1, 3}), PathIndex(12, kPath, 8, {1, 4})}); + EXPECT_NE(NullRegion(&file, 11, kPath).offset, NullRegion(&file, 12, kPath).offset); + EXPECT_EQ(ReadNulls(&file, 11, kPath), (std::vector {1, 3})); + EXPECT_EQ(ReadNulls(&file, 12, kPath), (std::vector {1, 4})); + } + // Same NULL rows, different document counts: the framed bitmap records the doc + // count, and the reader rejects a bitmap whose doc count is not the index's own. + { + MemoryFile file = + WriteContainer({PathIndex(11, kPath, 8, {1, 3}), PathIndex(12, kPath, 9, {1, 3})}); + EXPECT_NE(NullRegion(&file, 11, kPath).offset, NullRegion(&file, 12, kPath).offset); + EXPECT_EQ(ReadNulls(&file, 11, kPath), (std::vector {1, 3})); + EXPECT_EQ(ReadNulls(&file, 12, kPath), (std::vector {1, 3})); + } +} + +// Sharing is scoped to one suffix: indexes on different subcolumns keep their own +// sections contiguous even when their NULL rows happen to coincide. +TEST(SniiSharedNullBitmap, IdenticalBitmapsOnDifferentSubcolumnsAreNotShared) { + const std::vector nulls = {1, 3}; + MemoryFile file = + WriteContainer({PathIndex(11, kPath, 8, nulls), PathIndex(11, kOtherPath, 8, nulls)}); + const RegionRef first = NullRegion(&file, 11, kPath); + const RegionRef second = NullRegion(&file, 11, kOtherPath); + EXPECT_NE(first.offset, second.offset); + EXPECT_EQ(first.length, second.length); + EXPECT_LT(first.offset + first.length, second.offset); + EXPECT_EQ(ReadNulls(&file, 11, kPath), nulls); + EXPECT_EQ(ReadNulls(&file, 11, kOtherPath), nulls); +} + +// Compaction rebuilds every destination index through a streamed session. The +// sessions share the bitmap the same way, and each one still drops its bitmap bytes +// as soon as its session finishes: sharing retains nothing in memory. +TEST(SniiSharedNullBitmap, StreamedSessionsShareAndStillReleaseTheirBitmapBytes) { + const std::vector nulls = {0, 2, 7}; + writer::MemoryReporter reporter(nullptr, 1U << 20); + MemoryFile file; + SniiCompoundWriter compound(&file); + for (uint64_t index_id : {21, 22}) { + SniiIndexInput input = PathIndex(index_id, kPath, 8, nulls); + input.terms.clear(); + input.mem_reporter = &reporter; + SniiStreamedIndexSession* session = nullptr; + assert_ok(compound.begin_streamed_index(std::move(input), &session)); + ASSERT_NE(session, nullptr); + assert_ok(session->finish()); + EXPECT_EQ(reporter.current_bytes(), 0) << "index " << index_id; + } + assert_ok(compound.finish()); + + const RegionRef first = NullRegion(&file, 21, kPath); + const RegionRef second = NullRegion(&file, 22, kPath); + ASSERT_GT(first.length, 0U); + EXPECT_EQ(second.offset, first.offset); + EXPECT_EQ(second.length, first.length); + EXPECT_EQ(ReadNulls(&file, 21, kPath), nulls); + EXPECT_EQ(ReadNulls(&file, 22, kPath), nulls); +} + +// BUILD INDEX rewrites a container by copying its physical prefix and re-emitting +// the kept metadata groups. Dropping the index that wrote the shared region must not +// orphan the index that only references it. +TEST(SniiSharedNullBitmap, RewriteThatDropsTheWritingIndexKeepsTheSharedRegion) { + const std::vector nulls = {1, 2, 5}; + MemoryFile source = + WriteContainer({PathIndex(11, kPath, 8, nulls), PathIndex(12, kPath, 8, nulls)}); + const RegionRef shared = NullRegion(&source, 12, kPath); + ASSERT_EQ(shared.offset, NullRegion(&source, 11, kPath).offset); + + reader::SniiSegmentReader segment; + assert_ok(reader::SniiSegmentReader::open(&source, &segment)); + reader::SniiRewriteSnapshot snapshot; + assert_ok(segment.prepare_rewrite_snapshot( + {reader::LogicalIndexKey {.index_id = 12, .index_suffix = kPath}}, 8, &snapshot)); + EXPECT_GE(snapshot.physical_prefix_end(), shared.offset + shared.length); + + MemoryFile output; + SniiCompoundWriter compound(&output); + assert_ok(compound.inherit(snapshot, &source)); + // A new definition on the same subcolumn, added by the same rewrite. + assert_ok(compound.add_logical_index(PathIndex(13, kPath, 8, nulls))); + assert_ok(compound.finish()); + + reader::SniiSegmentReader rewritten; + assert_ok(reader::SniiSegmentReader::open(&output, &rewritten)); + EXPECT_EQ(rewritten.n_logical_indexes(), 2U); + const RegionRef kept = NullRegion(&output, 12, kPath); + EXPECT_EQ(kept.offset, shared.offset); + EXPECT_EQ(kept.length, shared.length); + EXPECT_EQ(ReadNulls(&output, 12, kPath), nulls); + EXPECT_EQ(ReadNulls(&output, 13, kPath), nulls); +} + +// Container bytes for N definitions on one subcolumn of a 1M-row segment whose NULL +// rows are pseudo-random (62.5% NULL, so the bitmap is made of bitset containers). +// The baseline puts the same N indexes on N distinct suffixes of equal length, which +// is exactly the layout before sharing: every index writes its own bitmap. +TEST(SniiSharedNullBitmap, SavesAllButOneCopyOfTheBitmap) { + constexpr uint32_t kDocCount = 1'000'000; + std::vector nulls; + for (uint32_t docid = 0; docid < kDocCount; ++docid) { + if (((docid * 2654435761U) >> 28) < 10) { + nulls.push_back(docid); + } + } + + for (size_t definitions : {2, 7}) { + std::vector shared_inputs; + std::vector separate_inputs; + for (size_t i = 0; i < definitions; ++i) { + shared_inputs.push_back(PathIndex(100 + i, kPath + "0", kDocCount, nulls)); + separate_inputs.push_back( + PathIndex(100 + i, kPath + std::to_string(i), kDocCount, nulls)); + } + MemoryFile shared = WriteContainer(shared_inputs); + MemoryFile separate = WriteContainer(separate_inputs); + + std::set shared_regions; + std::set separate_regions; + uint64_t bitmap_bytes = 0; + for (size_t i = 0; i < definitions; ++i) { + const RegionRef region = NullRegion(&shared, 100 + i, kPath + "0"); + shared_regions.insert(region.offset); + bitmap_bytes = region.length; + separate_regions.insert( + NullRegion(&separate, 100 + i, kPath + std::to_string(i)).offset); + } + EXPECT_EQ(shared_regions.size(), 1U); + EXPECT_EQ(separate_regions.size(), definitions); + EXPECT_EQ(ReadNulls(&shared, 100 + definitions - 1, kPath + "0"), nulls); + + const uint64_t expected_saving = (definitions - 1) * bitmap_bytes; + const uint64_t saving = separate.data().size() - shared.data().size(); + // Section offsets are varints in the metadata, and they shrink along with the + // container, so the measured saving can exceed the bitmap bytes by a few bytes. + EXPECT_GE(saving, expected_saving); + EXPECT_LE(saving, expected_saving + 16 * definitions); + std::cout << "[shared-null-bitmap] definitions=" << definitions + << " null_rows=" << nulls.size() << "/" << kDocCount + << " bitmap_bytes=" << bitmap_bytes + << " container_bytes_separate=" << separate.data().size() + << " container_bytes_shared=" << shared.data().size() << " saved=" << saving + << std::endl; + } +} diff --git a/be/test/storage/index/snii_writer_test.cpp b/be/test/storage/index/snii_writer_test.cpp index bae359c434d881..41ad5ab868d770 100644 --- a/be/test/storage/index/snii_writer_test.cpp +++ b/be/test/storage/index/snii_writer_test.cpp @@ -377,6 +377,49 @@ TEST(SniiWriterNorms, WritesNormsFollowSharedNormsPolicy) { doris::config::inverted_index_skip_norms_for_variant = original_skip_norms_for_variant; } +// Norms are only kept for rows that carry one: non-NULL rows (an empty value, an empty array and +// an array of NULL elements included, with length 0) and NULL ARRAY rows that still produced +// tokens. The writer keeps raw lengths until finish() so add_array_nulls can tell an empty NULL +// row from a NULL row with one token. +TEST(SniiWriterNorms, NullRowsKeepNormsOnlyWithTokens) { + doris::TabletIndexPB index_pb; + index_pb.set_index_type(doris::IndexType::INVERTED); + index_pb.set_index_id(93); + index_pb.set_index_name("norms_rows"); + index_pb.add_col_unique_id(0); + index_pb.mutable_properties()->insert({"parser", "english"}); + index_pb.mutable_properties()->insert({"support_phrase", "true"}); + doris::TabletIndex index_meta; + index_meta.init_from_pb(index_pb); + doris::segment_v2::SniiIndexColumnWriter writer(nullptr, &index_meta, + doris::FieldType::OLAP_FIELD_TYPE_VARCHAR); + ASSERT_OK(writer.init()); + ASSERT_TRUE(writer.writes_norms_for_test()); + + // docids 0-1: scalar values; 2-3: NULL scalars. + const std::vector values = {doris::Slice("alpha beta"), doris::Slice("")}; + ASSERT_OK(writer.add_values("", values.data(), values.size())); + ASSERT_OK(writer.add_nulls(2)); + + // docids 4-8: [alpha gamma delta], NULL row keeping [beta], NULL row, [], [NULL]. + const std::vector elements = {doris::Slice("alpha gamma delta"), + doris::Slice("beta"), doris::Slice("ignored")}; + const std::vector element_nulls = {0, 0, 1}; + const std::vector offsets = {0, 1, 2, 2, 2, 3}; + ASSERT_OK(writer.add_array_values(sizeof(doris::Slice), elements.data(), element_nulls.data(), + reinterpret_cast(offsets.data()), 5)); + EXPECT_EQ(writer.norm_lengths_for_test(), (std::vector {2, 0, 3, 1, 0, 0, 0})); + const std::vector null_map = {0, 1, 1, 0, 0}; + ASSERT_OK(writer.add_array_nulls(null_map.data(), null_map.size())); + + EXPECT_EQ(writer.null_docids_for_test(), (std::vector {2, 3, 5, 6})); + EXPECT_EQ(writer.null_docids_with_norms_for_test(), (std::vector {5})); + EXPECT_EQ(writer.norm_lengths_for_test(), (std::vector {2, 0, 3, 1, 0, 0})); + writer.close_on_error(); + EXPECT_TRUE(writer.norm_lengths_for_test().empty()); + EXPECT_TRUE(writer.null_docids_with_norms_for_test().empty()); +} + TEST(SniiDocIdSinkGrowth, AppendRangeGrowsGeometrically) { std::vector docids; doris::snii::query::VectorDocIdSink sink(docids); diff --git a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_norms.out b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_norms.out index 5f52500503f9cc..e81a537cb1ae52 100644 --- a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_norms.out +++ b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_norms.out @@ -4,8 +4,8 @@ 2 0.562 -- !snii_field_pattern_score -- -1 0.311 -2 0.5235 +1 0.1514 +2 0.2292 -- !snii_field_pattern_no_norms_match -- 1 diff --git a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_sparse_norms.out b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_sparse_norms.out new file mode 100644 index 00000000000000..a42ca901e3d297 --- /dev/null +++ b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_snii_sparse_norms.out @@ -0,0 +1,1177 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !adaptive_alpha_count -- +828 + +-- !adaptive_phrase_count -- +660 + +-- !adaptive_content_null -- +79132 + +-- !adaptive_not_beta -- +208 + +-- !adaptive_path_count -- +1600 + +-- !adaptive_path_phrase_count -- +1600 + +-- !adaptive_path_null -- +78400 + +-- !adaptive_alpha_scores -- +828 5 0.068345 0.482748 310.0080080000003 + +-- !adaptive_omega_scores -- +10022 5.088172 +12025 5.137658 +14028 5.175409 +16031 5.205156 +18034 5.229202 +20007 5.229202 +2010 4.48357 +22010 5.249041 +24013 5.265689 +26016 5.279859 +28019 5.292065 +30022 5.30269 +32025 5.312021 +34028 5.320282 +36031 5.327647 +38034 5.334254 +40007 5.340214 +4013 4.766776 +42010 5.345618 +44013 5.35054 +46016 5.355042 +48019 5.359176 +50022 5.362984 +52025 5.366504 +54028 5.369768 +56031 5.372802 +58034 5.37563 +60007 5.378273 +6016 4.922233 +62010 5.380746 +64013 5.383068 +66016 5.385251 +68019 5.387307 +7 3.805319 +70022 5.389246 +72025 5.391079 +74028 5.392815 +76031 5.39446 +78034 5.396022 +8019 5.020472 + +-- !adaptive_omega_phrase_scores -- +10022 6.850136 +12025 6.342776 +14028 5.905389 +16031 5.524434 +18034 5.189651 +20007 5.189651 +2010 10.073146 +22010 4.893126 +24013 4.628654 +26016 4.391306 +28019 4.177112 +30022 3.982842 +32025 3.805839 +34028 3.643899 +36031 3.495178 +38034 3.35812 +40007 3.231406 +4013 9.012986 +42010 3.113907 +44013 3.004653 +46016 2.902805 +48019 2.807636 +50022 2.718509 +52025 2.634867 +54028 2.556218 +56031 2.482128 +58034 2.412212 +60007 2.346126 +6016 8.154733 +62010 2.283566 +64013 2.224255 +66016 2.167947 +68019 2.114419 +70022 2.063471 +72025 2.01492 +74028 1.968602 +76031 1.924365 +78034 1.882073 +8019 7.44572 + +-- !adaptive_path_scores -- +1600 2 0.000723 0.000733 1.1647999999999885 + +-- !adaptive_rare_path_scores -- +10003 0.012384 +1003 0.012384 +11003 0.012384 +12003 0.012384 +13003 0.012384 +14003 0.012384 +15003 0.012384 +16003 0.012384 +17003 0.012384 +18003 0.012384 +19003 0.012384 +20003 0.012384 +2003 0.012384 +21003 0.012384 +22003 0.012384 +23003 0.012384 +24003 0.012384 +25003 0.012384 +26003 0.012384 +27003 0.012384 +28003 0.012384 +29003 0.012384 +3 0.012384 +30003 0.012384 +3003 0.012384 +31003 0.012384 +32003 0.012384 +33003 0.012384 +34003 0.012384 +35003 0.012384 +36003 0.012384 +37003 0.012384 +38003 0.012384 +39003 0.012384 +40003 0.012384 +4003 0.012384 +41003 0.012384 +42003 0.012384 +43003 0.012384 +44003 0.012384 +45003 0.012384 +46003 0.012384 +47003 0.012384 +48003 0.012384 +49003 0.012384 +50003 0.012384 +5003 0.012384 +51003 0.012384 +52003 0.012384 +53003 0.012384 +54003 0.012384 +55003 0.012384 +56003 0.012384 +57003 0.012384 +58003 0.012384 +59003 0.012384 +60003 0.012384 +6003 0.012384 +61003 0.012384 +62003 0.012384 +63003 0.012384 +64003 0.012384 +65003 0.012384 +66003 0.012384 +67003 0.012384 +68003 0.012384 +69003 0.012384 +70003 0.012384 +7003 0.012384 +71003 0.012384 +72003 0.012384 +73003 0.012384 +74003 0.012384 +75003 0.012384 +76003 0.012384 +77003 0.012384 +78003 0.012384 +79003 0.012384 +8003 0.012384 +9003 0.012384 + +-- !dense_alpha_count -- +828 + +-- !dense_phrase_count -- +660 + +-- !dense_content_null -- +79132 + +-- !dense_not_beta -- +208 + +-- !dense_path_count -- +1600 + +-- !dense_path_phrase_count -- +1600 + +-- !dense_path_null -- +78400 + +-- !dense_alpha_scores -- +828 5 0.068345 0.482748 310.0080080000003 + +-- !dense_omega_scores -- +10022 5.088172 +12025 5.137658 +14028 5.175409 +16031 5.205156 +18034 5.229202 +20007 5.229202 +2010 4.48357 +22010 5.249041 +24013 5.265689 +26016 5.279859 +28019 5.292065 +30022 5.30269 +32025 5.312021 +34028 5.320282 +36031 5.327647 +38034 5.334254 +40007 5.340214 +4013 4.766776 +42010 5.345618 +44013 5.35054 +46016 5.355042 +48019 5.359176 +50022 5.362984 +52025 5.366504 +54028 5.369768 +56031 5.372802 +58034 5.37563 +60007 5.378273 +6016 4.922233 +62010 5.380746 +64013 5.383068 +66016 5.385251 +68019 5.387307 +7 3.805319 +70022 5.389246 +72025 5.391079 +74028 5.392815 +76031 5.39446 +78034 5.396022 +8019 5.020472 + +-- !dense_omega_phrase_scores -- +10022 6.850136 +12025 6.342776 +14028 5.905389 +16031 5.524434 +18034 5.189651 +20007 5.189651 +2010 10.073146 +22010 4.893126 +24013 4.628654 +26016 4.391306 +28019 4.177112 +30022 3.982842 +32025 3.805839 +34028 3.643899 +36031 3.495178 +38034 3.35812 +40007 3.231406 +4013 9.012986 +42010 3.113907 +44013 3.004653 +46016 2.902805 +48019 2.807636 +50022 2.718509 +52025 2.634867 +54028 2.556218 +56031 2.482128 +58034 2.412212 +60007 2.346126 +6016 8.154733 +62010 2.283566 +64013 2.224255 +66016 2.167947 +68019 2.114419 +70022 2.063471 +72025 2.01492 +74028 1.968602 +76031 1.924365 +78034 1.882073 +8019 7.44572 + +-- !dense_path_scores -- +1600 2 0.000723 0.000733 1.1647999999999885 + +-- !dense_rare_path_scores -- +10003 0.012384 +1003 0.012384 +11003 0.012384 +12003 0.012384 +13003 0.012384 +14003 0.012384 +15003 0.012384 +16003 0.012384 +17003 0.012384 +18003 0.012384 +19003 0.012384 +20003 0.012384 +2003 0.012384 +21003 0.012384 +22003 0.012384 +23003 0.012384 +24003 0.012384 +25003 0.012384 +26003 0.012384 +27003 0.012384 +28003 0.012384 +29003 0.012384 +3 0.012384 +30003 0.012384 +3003 0.012384 +31003 0.012384 +32003 0.012384 +33003 0.012384 +34003 0.012384 +35003 0.012384 +36003 0.012384 +37003 0.012384 +38003 0.012384 +39003 0.012384 +40003 0.012384 +4003 0.012384 +41003 0.012384 +42003 0.012384 +43003 0.012384 +44003 0.012384 +45003 0.012384 +46003 0.012384 +47003 0.012384 +48003 0.012384 +49003 0.012384 +50003 0.012384 +5003 0.012384 +51003 0.012384 +52003 0.012384 +53003 0.012384 +54003 0.012384 +55003 0.012384 +56003 0.012384 +57003 0.012384 +58003 0.012384 +59003 0.012384 +60003 0.012384 +6003 0.012384 +61003 0.012384 +62003 0.012384 +63003 0.012384 +64003 0.012384 +65003 0.012384 +66003 0.012384 +67003 0.012384 +68003 0.012384 +69003 0.012384 +70003 0.012384 +7003 0.012384 +71003 0.012384 +72003 0.012384 +73003 0.012384 +74003 0.012384 +75003 0.012384 +76003 0.012384 +77003 0.012384 +78003 0.012384 +79003 0.012384 +8003 0.012384 +9003 0.012384 + +-- !mixed_alpha_count -- +828 + +-- !mixed_phrase_count -- +660 + +-- !mixed_content_null -- +79132 + +-- !mixed_not_beta -- +208 + +-- !mixed_path_count -- +1600 + +-- !mixed_path_phrase_count -- +1600 + +-- !mixed_path_null -- +78400 + +-- !mixed_alpha_scores -- +828 5 0.068345 0.482748 310.0080080000003 + +-- !mixed_omega_scores -- +10022 5.088172 +12025 5.137658 +14028 5.175409 +16031 5.205156 +18034 5.229202 +20007 5.229202 +2010 4.48357 +22010 5.249041 +24013 5.265689 +26016 5.279859 +28019 5.292065 +30022 5.30269 +32025 5.312021 +34028 5.320282 +36031 5.327647 +38034 5.334254 +40007 5.340214 +4013 4.766776 +42010 5.345618 +44013 5.35054 +46016 5.355042 +48019 5.359176 +50022 5.362984 +52025 5.366504 +54028 5.369768 +56031 5.372802 +58034 5.37563 +60007 5.378273 +6016 4.922233 +62010 5.380746 +64013 5.383068 +66016 5.385251 +68019 5.387307 +7 3.805319 +70022 5.389246 +72025 5.391079 +74028 5.392815 +76031 5.39446 +78034 5.396022 +8019 5.020472 + +-- !mixed_omega_phrase_scores -- +10022 6.850136 +12025 6.342776 +14028 5.905389 +16031 5.524434 +18034 5.189651 +20007 5.189651 +2010 10.073146 +22010 4.893126 +24013 4.628654 +26016 4.391306 +28019 4.177112 +30022 3.982842 +32025 3.805839 +34028 3.643899 +36031 3.495178 +38034 3.35812 +40007 3.231406 +4013 9.012986 +42010 3.113907 +44013 3.004653 +46016 2.902805 +48019 2.807636 +50022 2.718509 +52025 2.634867 +54028 2.556218 +56031 2.482128 +58034 2.412212 +60007 2.346126 +6016 8.154733 +62010 2.283566 +64013 2.224255 +66016 2.167947 +68019 2.114419 +70022 2.063471 +72025 2.01492 +74028 1.968602 +76031 1.924365 +78034 1.882073 +8019 7.44572 + +-- !mixed_path_scores -- +1600 2 0.000723 0.000733 1.1647999999999885 + +-- !mixed_rare_path_scores -- +10003 0.012384 +1003 0.012384 +11003 0.012384 +12003 0.012384 +13003 0.012384 +14003 0.012384 +15003 0.012384 +16003 0.012384 +17003 0.012384 +18003 0.012384 +19003 0.012384 +20003 0.012384 +2003 0.012384 +21003 0.012384 +22003 0.012384 +23003 0.012384 +24003 0.012384 +25003 0.012384 +26003 0.012384 +27003 0.012384 +28003 0.012384 +29003 0.012384 +3 0.012384 +30003 0.012384 +3003 0.012384 +31003 0.012384 +32003 0.012384 +33003 0.012384 +34003 0.012384 +35003 0.012384 +36003 0.012384 +37003 0.012384 +38003 0.012384 +39003 0.012384 +40003 0.012384 +4003 0.012384 +41003 0.012384 +42003 0.012384 +43003 0.012384 +44003 0.012384 +45003 0.012384 +46003 0.012384 +47003 0.012384 +48003 0.012384 +49003 0.012384 +50003 0.012384 +5003 0.012384 +51003 0.012384 +52003 0.012384 +53003 0.012384 +54003 0.012384 +55003 0.012384 +56003 0.012384 +57003 0.012384 +58003 0.012384 +59003 0.012384 +60003 0.012384 +6003 0.012384 +61003 0.012384 +62003 0.012384 +63003 0.012384 +64003 0.012384 +65003 0.012384 +66003 0.012384 +67003 0.012384 +68003 0.012384 +69003 0.012384 +70003 0.012384 +7003 0.012384 +71003 0.012384 +72003 0.012384 +73003 0.012384 +74003 0.012384 +75003 0.012384 +76003 0.012384 +77003 0.012384 +78003 0.012384 +79003 0.012384 +8003 0.012384 +9003 0.012384 + +-- !adaptive_vs_dense_content_score_diff -- +0 + +-- !adaptive_vs_dense_path_score_diff -- +0 + +-- !mixed_vs_dense_content_score_diff -- +0 + +-- !mixed_vs_dense_path_score_diff -- +0 + +-- !adaptive_compacted_alpha_count -- +828 + +-- !adaptive_compacted_phrase_count -- +660 + +-- !adaptive_compacted_content_null -- +79132 + +-- !adaptive_compacted_not_beta -- +208 + +-- !adaptive_compacted_path_count -- +1600 + +-- !adaptive_compacted_path_phrase_count -- +1600 + +-- !adaptive_compacted_path_null -- +78400 + +-- !adaptive_compacted_alpha_scores -- +828 5 0.068345 0.482748 310.0080080000003 + +-- !adaptive_compacted_omega_scores -- +10022 5.088172 +12025 5.137658 +14028 5.175409 +16031 5.205156 +18034 5.229202 +20007 5.229202 +2010 4.48357 +22010 5.249041 +24013 5.265689 +26016 5.279859 +28019 5.292065 +30022 5.30269 +32025 5.312021 +34028 5.320282 +36031 5.327647 +38034 5.334254 +40007 5.340214 +4013 4.766776 +42010 5.345618 +44013 5.35054 +46016 5.355042 +48019 5.359176 +50022 5.362984 +52025 5.366504 +54028 5.369768 +56031 5.372802 +58034 5.37563 +60007 5.378273 +6016 4.922233 +62010 5.380746 +64013 5.383068 +66016 5.385251 +68019 5.387307 +7 3.805319 +70022 5.389246 +72025 5.391079 +74028 5.392815 +76031 5.39446 +78034 5.396022 +8019 5.020472 + +-- !adaptive_compacted_omega_phrase_scores -- +10022 6.850136 +12025 6.342776 +14028 5.905389 +16031 5.524434 +18034 5.189651 +20007 5.189651 +2010 10.073146 +22010 4.893126 +24013 4.628654 +26016 4.391306 +28019 4.177112 +30022 3.982842 +32025 3.805839 +34028 3.643899 +36031 3.495178 +38034 3.35812 +40007 3.231406 +4013 9.012986 +42010 3.113907 +44013 3.004653 +46016 2.902805 +48019 2.807636 +50022 2.718509 +52025 2.634867 +54028 2.556218 +56031 2.482128 +58034 2.412212 +60007 2.346126 +6016 8.154733 +62010 2.283566 +64013 2.224255 +66016 2.167947 +68019 2.114419 +70022 2.063471 +72025 2.01492 +74028 1.968602 +76031 1.924365 +78034 1.882073 +8019 7.44572 + +-- !adaptive_compacted_path_scores -- +1600 2 0.000723 0.000733 1.1647999999999885 + +-- !adaptive_compacted_rare_path_scores -- +10003 0.012384 +1003 0.012384 +11003 0.012384 +12003 0.012384 +13003 0.012384 +14003 0.012384 +15003 0.012384 +16003 0.012384 +17003 0.012384 +18003 0.012384 +19003 0.012384 +20003 0.012384 +2003 0.012384 +21003 0.012384 +22003 0.012384 +23003 0.012384 +24003 0.012384 +25003 0.012384 +26003 0.012384 +27003 0.012384 +28003 0.012384 +29003 0.012384 +3 0.012384 +30003 0.012384 +3003 0.012384 +31003 0.012384 +32003 0.012384 +33003 0.012384 +34003 0.012384 +35003 0.012384 +36003 0.012384 +37003 0.012384 +38003 0.012384 +39003 0.012384 +40003 0.012384 +4003 0.012384 +41003 0.012384 +42003 0.012384 +43003 0.012384 +44003 0.012384 +45003 0.012384 +46003 0.012384 +47003 0.012384 +48003 0.012384 +49003 0.012384 +50003 0.012384 +5003 0.012384 +51003 0.012384 +52003 0.012384 +53003 0.012384 +54003 0.012384 +55003 0.012384 +56003 0.012384 +57003 0.012384 +58003 0.012384 +59003 0.012384 +60003 0.012384 +6003 0.012384 +61003 0.012384 +62003 0.012384 +63003 0.012384 +64003 0.012384 +65003 0.012384 +66003 0.012384 +67003 0.012384 +68003 0.012384 +69003 0.012384 +70003 0.012384 +7003 0.012384 +71003 0.012384 +72003 0.012384 +73003 0.012384 +74003 0.012384 +75003 0.012384 +76003 0.012384 +77003 0.012384 +78003 0.012384 +79003 0.012384 +8003 0.012384 +9003 0.012384 + +-- !dense_compacted_alpha_count -- +828 + +-- !dense_compacted_phrase_count -- +660 + +-- !dense_compacted_content_null -- +79132 + +-- !dense_compacted_not_beta -- +208 + +-- !dense_compacted_path_count -- +1600 + +-- !dense_compacted_path_phrase_count -- +1600 + +-- !dense_compacted_path_null -- +78400 + +-- !dense_compacted_alpha_scores -- +828 5 0.068345 0.482748 310.0080080000003 + +-- !dense_compacted_omega_scores -- +10022 5.088172 +12025 5.137658 +14028 5.175409 +16031 5.205156 +18034 5.229202 +20007 5.229202 +2010 4.48357 +22010 5.249041 +24013 5.265689 +26016 5.279859 +28019 5.292065 +30022 5.30269 +32025 5.312021 +34028 5.320282 +36031 5.327647 +38034 5.334254 +40007 5.340214 +4013 4.766776 +42010 5.345618 +44013 5.35054 +46016 5.355042 +48019 5.359176 +50022 5.362984 +52025 5.366504 +54028 5.369768 +56031 5.372802 +58034 5.37563 +60007 5.378273 +6016 4.922233 +62010 5.380746 +64013 5.383068 +66016 5.385251 +68019 5.387307 +7 3.805319 +70022 5.389246 +72025 5.391079 +74028 5.392815 +76031 5.39446 +78034 5.396022 +8019 5.020472 + +-- !dense_compacted_omega_phrase_scores -- +10022 6.850136 +12025 6.342776 +14028 5.905389 +16031 5.524434 +18034 5.189651 +20007 5.189651 +2010 10.073146 +22010 4.893126 +24013 4.628654 +26016 4.391306 +28019 4.177112 +30022 3.982842 +32025 3.805839 +34028 3.643899 +36031 3.495178 +38034 3.35812 +40007 3.231406 +4013 9.012986 +42010 3.113907 +44013 3.004653 +46016 2.902805 +48019 2.807636 +50022 2.718509 +52025 2.634867 +54028 2.556218 +56031 2.482128 +58034 2.412212 +60007 2.346126 +6016 8.154733 +62010 2.283566 +64013 2.224255 +66016 2.167947 +68019 2.114419 +70022 2.063471 +72025 2.01492 +74028 1.968602 +76031 1.924365 +78034 1.882073 +8019 7.44572 + +-- !dense_compacted_path_scores -- +1600 2 0.000723 0.000733 1.1647999999999885 + +-- !dense_compacted_rare_path_scores -- +10003 0.012384 +1003 0.012384 +11003 0.012384 +12003 0.012384 +13003 0.012384 +14003 0.012384 +15003 0.012384 +16003 0.012384 +17003 0.012384 +18003 0.012384 +19003 0.012384 +20003 0.012384 +2003 0.012384 +21003 0.012384 +22003 0.012384 +23003 0.012384 +24003 0.012384 +25003 0.012384 +26003 0.012384 +27003 0.012384 +28003 0.012384 +29003 0.012384 +3 0.012384 +30003 0.012384 +3003 0.012384 +31003 0.012384 +32003 0.012384 +33003 0.012384 +34003 0.012384 +35003 0.012384 +36003 0.012384 +37003 0.012384 +38003 0.012384 +39003 0.012384 +40003 0.012384 +4003 0.012384 +41003 0.012384 +42003 0.012384 +43003 0.012384 +44003 0.012384 +45003 0.012384 +46003 0.012384 +47003 0.012384 +48003 0.012384 +49003 0.012384 +50003 0.012384 +5003 0.012384 +51003 0.012384 +52003 0.012384 +53003 0.012384 +54003 0.012384 +55003 0.012384 +56003 0.012384 +57003 0.012384 +58003 0.012384 +59003 0.012384 +60003 0.012384 +6003 0.012384 +61003 0.012384 +62003 0.012384 +63003 0.012384 +64003 0.012384 +65003 0.012384 +66003 0.012384 +67003 0.012384 +68003 0.012384 +69003 0.012384 +70003 0.012384 +7003 0.012384 +71003 0.012384 +72003 0.012384 +73003 0.012384 +74003 0.012384 +75003 0.012384 +76003 0.012384 +77003 0.012384 +78003 0.012384 +79003 0.012384 +8003 0.012384 +9003 0.012384 + +-- !mixed_compacted_alpha_count -- +828 + +-- !mixed_compacted_phrase_count -- +660 + +-- !mixed_compacted_content_null -- +79132 + +-- !mixed_compacted_not_beta -- +208 + +-- !mixed_compacted_path_count -- +1600 + +-- !mixed_compacted_path_phrase_count -- +1600 + +-- !mixed_compacted_path_null -- +78400 + +-- !mixed_compacted_alpha_scores -- +828 5 0.068345 0.482748 310.0080080000003 + +-- !mixed_compacted_omega_scores -- +10022 5.088172 +12025 5.137658 +14028 5.175409 +16031 5.205156 +18034 5.229202 +20007 5.229202 +2010 4.48357 +22010 5.249041 +24013 5.265689 +26016 5.279859 +28019 5.292065 +30022 5.30269 +32025 5.312021 +34028 5.320282 +36031 5.327647 +38034 5.334254 +40007 5.340214 +4013 4.766776 +42010 5.345618 +44013 5.35054 +46016 5.355042 +48019 5.359176 +50022 5.362984 +52025 5.366504 +54028 5.369768 +56031 5.372802 +58034 5.37563 +60007 5.378273 +6016 4.922233 +62010 5.380746 +64013 5.383068 +66016 5.385251 +68019 5.387307 +7 3.805319 +70022 5.389246 +72025 5.391079 +74028 5.392815 +76031 5.39446 +78034 5.396022 +8019 5.020472 + +-- !mixed_compacted_omega_phrase_scores -- +10022 6.850136 +12025 6.342776 +14028 5.905389 +16031 5.524434 +18034 5.189651 +20007 5.189651 +2010 10.073146 +22010 4.893126 +24013 4.628654 +26016 4.391306 +28019 4.177112 +30022 3.982842 +32025 3.805839 +34028 3.643899 +36031 3.495178 +38034 3.35812 +40007 3.231406 +4013 9.012986 +42010 3.113907 +44013 3.004653 +46016 2.902805 +48019 2.807636 +50022 2.718509 +52025 2.634867 +54028 2.556218 +56031 2.482128 +58034 2.412212 +60007 2.346126 +6016 8.154733 +62010 2.283566 +64013 2.224255 +66016 2.167947 +68019 2.114419 +70022 2.063471 +72025 2.01492 +74028 1.968602 +76031 1.924365 +78034 1.882073 +8019 7.44572 + +-- !mixed_compacted_path_scores -- +1600 2 0.000723 0.000733 1.1647999999999885 + +-- !mixed_compacted_rare_path_scores -- +10003 0.012384 +1003 0.012384 +11003 0.012384 +12003 0.012384 +13003 0.012384 +14003 0.012384 +15003 0.012384 +16003 0.012384 +17003 0.012384 +18003 0.012384 +19003 0.012384 +20003 0.012384 +2003 0.012384 +21003 0.012384 +22003 0.012384 +23003 0.012384 +24003 0.012384 +25003 0.012384 +26003 0.012384 +27003 0.012384 +28003 0.012384 +29003 0.012384 +3 0.012384 +30003 0.012384 +3003 0.012384 +31003 0.012384 +32003 0.012384 +33003 0.012384 +34003 0.012384 +35003 0.012384 +36003 0.012384 +37003 0.012384 +38003 0.012384 +39003 0.012384 +40003 0.012384 +4003 0.012384 +41003 0.012384 +42003 0.012384 +43003 0.012384 +44003 0.012384 +45003 0.012384 +46003 0.012384 +47003 0.012384 +48003 0.012384 +49003 0.012384 +50003 0.012384 +5003 0.012384 +51003 0.012384 +52003 0.012384 +53003 0.012384 +54003 0.012384 +55003 0.012384 +56003 0.012384 +57003 0.012384 +58003 0.012384 +59003 0.012384 +60003 0.012384 +6003 0.012384 +61003 0.012384 +62003 0.012384 +63003 0.012384 +64003 0.012384 +65003 0.012384 +66003 0.012384 +67003 0.012384 +68003 0.012384 +69003 0.012384 +70003 0.012384 +7003 0.012384 +71003 0.012384 +72003 0.012384 +73003 0.012384 +74003 0.012384 +75003 0.012384 +76003 0.012384 +77003 0.012384 +78003 0.012384 +79003 0.012384 +8003 0.012384 +9003 0.012384 + +-- !compacted_adaptive_vs_dense_content_score_diff -- +0 + +-- !compacted_adaptive_vs_dense_path_score_diff -- +0 + +-- !compacted_mixed_vs_dense_content_score_diff -- +0 + +-- !compacted_mixed_vs_dense_path_score_diff -- +0 + diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_sparse_norms.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_sparse_norms.groovy new file mode 100644 index 00000000000000..853d64682519e3 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_snii_sparse_norms.groovy @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SNII keeps BM25 norms only for the rows that carry one when that is smaller than a byte per row +// (the sparse norms layout). Mostly NULL columns and VARIANT paths are exactly that case. The BE +// config enable_snii_sparse_norms (default true) allows that layout; turned off, the writer emits +// the dense layout every earlier SNII writer produced. This suite loads the same batches three +// ways -- config on, config off, and toggled between batches (old and new layouts side by side) -- +// and checks that MATCH, IS NULL and score() agree, before and after a full compaction, which +// runs with the config off for the dense table and on for the others. +// Scores use per-field document counts: N and avgdl count the non-NULL rows of the field. +// It changes a BE config, so it must not share the cluster with other suites. +suite("test_storage_format_snii_sparse_norms", "p0,nonConcurrent") { + sql """ set enable_match_without_inverted_index = false """ + sql """ set default_variant_enable_typed_paths_to_sparse = false """ + sql """ set default_variant_enable_doc_mode = false """ + + def tables = ["snii_sparse_norms_adaptive", "snii_sparse_norms_dense", "snii_sparse_norms_mixed"] + + def createTable = { String table -> + sql "DROP TABLE IF EXISTS ${table}" + sql """ + CREATE TABLE ${table} ( + id INT, + content TEXT NULL, + v variant< + 's_*' : text, + PROPERTIES("variant_max_subcolumns_count"="0") + > NULL, + INDEX idx_content (content) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true" + ), + INDEX idx_v (v) USING INVERTED PROPERTIES( + "parser"="english", + "support_phrase"="true", + "field_pattern"="s_*" + ) + ) ENGINE=OLAP DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true", + "inverted_index_storage_format" = "SNII" + ) + """ + } + + // One batch = one segment of 20000 rows. content is set on about 1% of the rows; each s_N + // path on 2%, s_rare on 0.1%. "omega" occurs a different number of times in each row that + // has it, so its scores are distinct. + def loadBatch = { String table, int batch -> + def offset = batch * 20000 + sql """ + INSERT INTO ${table} + SELECT number + ${offset}, + CASE WHEN number % 97 = 0 THEN concat('alpha ', repeat('beta ', number % 5)) + WHEN number % 2003 = 7 THEN concat( + repeat('omega ', cast(floor((number + ${offset}) / 2003) + 1 as int)), + 'tail') + ELSE NULL END, + parse_to_variant(concat('{"s_', number % 50, '":"delta ', repeat('eps ', number % 4), + '"', if(number % 1000 = 3, ',"s_rare":"omega delta omega"', ''), '}')) + FROM numbers("number" = "20000") + """ + } + + // Loads one batch with enable_snii_sparse_norms set explicitly; the BE value is restored after. + def loadWithSparseNorms = { String table, int batch, boolean sparseNorms -> + setBeConfigTemporary([enable_snii_sparse_norms: sparseNorms]) { + loadBatch(table, batch) + } + } + + def checkTable = { String tag, String table -> + "order_qt_${tag}_alpha_count" """ + select count(*) from ${table} where content match_any 'alpha' + """ + "order_qt_${tag}_phrase_count" """ + select count(*) from ${table} where content match_phrase 'alpha beta' + """ + "order_qt_${tag}_content_null" """ + select count(*) from ${table} where content is null + """ + "order_qt_${tag}_not_beta" """ + select count(*) from ${table} where not (content match_any 'beta') + """ + "order_qt_${tag}_path_count" """ + select count(*) from ${table} where cast(v['s_7'] as string) match_any 'eps' + """ + "order_qt_${tag}_path_phrase_count" """ + select count(*) from ${table} where cast(v['s_7'] as string) match_phrase 'delta eps' + """ + "order_qt_${tag}_path_null" """ + select count(*) from ${table} where cast(v['s_7'] as string) is null + """ + "order_qt_${tag}_alpha_scores" """ + select count(*), count(distinct s), min(s), max(s), sum(s) from ( + select id, round(score(), 6) as s from ${table} + where content match_any 'alpha beta' order by score() desc limit 100000) t + """ + "order_qt_${tag}_omega_scores" """ + select id, round(score(), 6) from ${table} + where content match_any 'omega' order by score() desc limit 100 + """ + "order_qt_${tag}_omega_phrase_scores" """ + select id, round(score(), 6) from ${table} + where content match_phrase 'omega omega tail' order by score() desc limit 100 + """ + "order_qt_${tag}_path_scores" """ + select count(*), count(distinct s), min(s), max(s), sum(s) from ( + select id, round(score(), 6) as s from ${table} + where cast(v['s_7'] as string) match_any 'delta eps' order by score() desc limit 100000) t + """ + "order_qt_${tag}_rare_path_scores" """ + select id, round(score(), 6) from ${table} + where cast(v['s_rare'] as string) match_phrase 'omega delta' order by score() desc limit 100 + """ + } + + // Rows whose score differs between two tables (exact, not rounded). + def checkSameScores = { String tag, String left, String right -> + "order_qt_${tag}_content_score_diff" """ + select count(*) from ( + select id, score() as s from ${left} + where content match_any 'alpha beta omega' order by score() desc limit 100000) a + full outer join ( + select id, score() as s from ${right} + where content match_any 'alpha beta omega' order by score() desc limit 100000) b + on a.id = b.id + where a.id is null or b.id is null or a.s != b.s + """ + "order_qt_${tag}_path_score_diff" """ + select count(*) from ( + select id, score() as s from ${left} + where cast(v['s_3'] as string) match_phrase 'delta eps' order by score() desc limit 100000) a + full outer join ( + select id, score() as s from ${right} + where cast(v['s_3'] as string) match_phrase 'delta eps' order by score() desc limit 100000) b + on a.id = b.id + where a.id is null or b.id is null or a.s != b.s + """ + } + + def indexDiskSize = { String table -> + def tablets = sql_return_maparray """show tablets from ${table}""" + def size = 0L + for (tablet in tablets) { + def rows = sql """ + select sum(INDEX_DISK_SIZE) from information_schema.rowsets + where TABLET_ID = ${tablet.TabletId} + """ + size += rows[0][0] as long + } + return size + } + + for (table in tables) { + createTable(table) + } + for (int batch = 0; batch < 4; batch++) { + loadWithSparseNorms("snii_sparse_norms_adaptive", batch, true) + loadWithSparseNorms("snii_sparse_norms_dense", batch, false) + // Old-layout and new-layout segments side by side. + loadWithSparseNorms("snii_sparse_norms_mixed", batch, batch % 2 == 1) + } + sql " sync " + + // Only the norms layout differs between the tables, so the sparse layout makes them smaller. + def adaptiveSize = indexDiskSize("snii_sparse_norms_adaptive") + def denseSize = indexDiskSize("snii_sparse_norms_dense") + def mixedSize = indexDiskSize("snii_sparse_norms_mixed") + logger.info("SNII index sizes: adaptive=${adaptiveSize}, dense=${denseSize}, mixed=${mixedSize}") + assertTrue(adaptiveSize < mixedSize) + assertTrue(mixedSize < denseSize) + + checkTable("adaptive", "snii_sparse_norms_adaptive") + checkTable("dense", "snii_sparse_norms_dense") + checkTable("mixed", "snii_sparse_norms_mixed") + checkSameScores("adaptive_vs_dense", "snii_sparse_norms_adaptive", "snii_sparse_norms_dense") + checkSameScores("mixed_vs_dense", "snii_sparse_norms_mixed", "snii_sparse_norms_dense") + + // Full compaction: the dense table compacts with the config off, the others with it on; every + // source mix must merge into the same answers. + setBeConfigTemporary([enable_snii_sparse_norms: true]) { + trigger_and_wait_compaction("snii_sparse_norms_adaptive", "full", 1800) + trigger_and_wait_compaction("snii_sparse_norms_mixed", "full", 1800) + } + setBeConfigTemporary([enable_snii_sparse_norms: false]) { + trigger_and_wait_compaction("snii_sparse_norms_dense", "full", 1800) + } + + checkTable("adaptive_compacted", "snii_sparse_norms_adaptive") + checkTable("dense_compacted", "snii_sparse_norms_dense") + checkTable("mixed_compacted", "snii_sparse_norms_mixed") + checkSameScores("compacted_adaptive_vs_dense", "snii_sparse_norms_adaptive", + "snii_sparse_norms_dense") + checkSameScores("compacted_mixed_vs_dense", "snii_sparse_norms_mixed", + "snii_sparse_norms_dense") + // The compaction output follows the config it ran with, whatever layouts its sources had. + def compactedDenseSize = indexDiskSize("snii_sparse_norms_dense") + def compactedAdaptiveSize = indexDiskSize("snii_sparse_norms_adaptive") + def compactedMixedSize = indexDiskSize("snii_sparse_norms_mixed") + logger.info("SNII compacted index sizes: adaptive=${compactedAdaptiveSize}, " + + "dense=${compactedDenseSize}, mixed=${compactedMixedSize}") + assertTrue(compactedAdaptiveSize < compactedDenseSize) + assertTrue(compactedMixedSize < compactedDenseSize) +}