diff --git a/changelog/unreleased/SOLR-18201-range-field-docvalues-optimization.yml b/changelog/unreleased/SOLR-18201-range-field-docvalues-optimization.yml new file mode 100644 index 000000000000..ab6aa7627bd1 --- /dev/null +++ b/changelog/unreleased/SOLR-18201-range-field-docvalues-optimization.yml @@ -0,0 +1,10 @@ +title: > + Range field types (Int/Long/Float/DoubleRangeField) now support docValues="true" + as a query-time filter optimization. +type: added +authors: + - name: Sonu Sharma + nick: ercsonusharma +links: + - name: SOLR-18201 + url: https://issues.apache.org/jira/browse/SOLR-18201 diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/RangeSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/RangeSearch.java new file mode 100644 index 000000000000..95456badb8c7 --- /dev/null +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/RangeSearch.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.search; + +import static org.apache.solr.bench.BaseBenchState.log; +import static org.apache.solr.bench.Docs.docs; +import static org.apache.solr.bench.generators.SourceDSL.integers; +import static org.apache.solr.bench.generators.SourceDSL.lists; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.solr.bench.CircularIterator; +import org.apache.solr.bench.Docs; +import org.apache.solr.bench.SolrBenchState; +import org.apache.solr.bench.generators.SolrGen; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmarks the docValues filter optimization for range field types. + * + *

Docs hold wide ranges {@code [m, m+SPAN]} in indexed-only (BKD) and indexed+docValues fields, + * single-valued ({@code r_ir}/{@code r_ir_dv}) and multiValued ({@code r_mv_ir}/{@code + * r_mv_ir_dv}). A central window matches ~half the docs, so the range clause is costly to + * materialize but cheap to verify per-doc. It is AND-ed with a selective term filter whose + * doc-frequency is controlled, so we know and the {@code [range-bench]} setup log confirms, which + * {@link org.apache.lucene.search.IndexOrDocValuesQuery} branch runs. + * + *

Params: {@code criteria} (the {@code {!numericRange criteria=...}} relation), {@code field} + * (the DV / non-DV pairs), and {@code scenario} - the lead's selectivity vs the {@code rangeCost/8} + * cutover: {@code selective_dv} (~{@value #SELECTIVE_LEAD_DF} docs -> docValues branch), {@code + * broad_points} (~{@value #BROAD_LEAD_DF} docs -> points branch), {@code none_points} + * (standalone -> points). + */ +@Fork(value = 1) +@Warmup(time = 5, iterations = 5) +@Measurement(time = 5, iterations = 5) +@Threads(value = 1) +public class RangeSearch { + + static final String COLLECTION = "c1"; + + static final int NUM_DOCS = 1_000_000; + + /** Width of each indexed document range: docs store {@code [m, m+SPAN]}. */ + static final int SPAN = 500_000; + + /** + * Narrow window for {@code contains}/{@code intersects}/{@code crosses}: contained by ~half the + * docs. + */ + static final String OTHER_QUERY = "[500000 TO 500100]"; + + /** + * Wide window for {@code within} (wider than {@code SPAN}), so ~half the docs' ranges fit inside. + */ + static final String WITHIN_QUERY = "[0 TO 1000000]"; + + /** "Selective" lead df: far below {@code rangeCost/8} (~62.5k) -> docValues branch. */ + static final int SELECTIVE_LEAD_DF = 2_000; + + /** "Broad" lead df: above {@code rangeCost/8} but below {@code rangeCost} -> points branch. */ + static final int BROAD_LEAD_DF = 250_000; + + @State(Scope.Benchmark) + public static class BenchState { + + @Param({"contains", "intersects", "within", "crosses"}) + public String criteria; + + @Param({"r_ir", "r_ir_dv", "r_mv_ir", "r_mv_ir_dv"}) + public String field; + + @Param({"selective_dv", "broad_points", "none_points"}) + public String scenario; + + // Rotated so repeated queries are distinct top-level queries (each ~SELECTIVE_LEAD_DF docs). + Iterator selectiveTerms; + // Rotated broad-lead terms (each ~BROAD_LEAD_DF docs). + Iterator broadTerms; + + @Setup(Level.Trial) + public void setupTrial(SolrBenchState solrBenchState) throws Exception { + solrBenchState.setUseHttp1(true); + solrBenchState.startSolr(1); + solrBenchState.createCollection(COLLECTION, 1, 1); + + // Controlled lead selectivity: ~numDocs/DF distinct values -> each term matches ~DF docs. + SolrGen selectiveLeadTerms = + integers().between(0, NUM_DOCS / SELECTIVE_LEAD_DF - 1).map(i -> "s" + i, String.class); + SolrGen broadLeadTerms = + integers().between(0, NUM_DOCS / BROAD_LEAD_DF - 1).map(i -> "b" + i, String.class); + // m in [0, NUM_DOCS] so that m + SPAN never overflows int; range string is "[m TO m+SPAN]". + SolrGen rangeValues = + integers() + .between(0, NUM_DOCS) + .map(m -> "[" + m + " TO " + (m + SPAN) + "]", String.class); + // multiValued docs hold 2-3 ranges each, drawn from the same distribution. + var mvRangeValues = lists().of(rangeValues).ofSizeBetween(2, 3); + + Docs docs = + docs() + .field("id", integers().incrementing()) + .field("term_high_s", selectiveLeadTerms) + .field("term_low_s", broadLeadTerms) + .field("r_ir", rangeValues) // single-valued, indexed-only (BKD) + .field("r_ir_dv", rangeValues) // single-valued, indexed + docValues + .field("r_mv_ir", mvRangeValues) // multiValued, indexed-only (BKD) + .field("r_mv_ir_dv", mvRangeValues); // multiValued, indexed + docValues + + solrBenchState.index(COLLECTION, docs, NUM_DOCS, false); + + // Discover the real term values (and their counts) so queries hit existing terms. + String basePath = solrBenchState.nodes.getFirst(); + SolrQuery q = new SolrQuery("*:*"); + q.setParam("facet", "true"); + q.setParam("rows", "0"); + q.setParam("facet.field", "term_high_s", "term_low_s"); + q.setParam("facet.limit", "2000"); + q.setParam("facet.mincount", "1"); + QueryResponse response = + new QueryRequest(q).processWithBaseUrl(solrBenchState.client, basePath, COLLECTION); + selectiveTerms = new CircularIterator<>(facetValues(response, "term_high_s")); + broadTerms = new CircularIterator<>(facetValues(response, "term_low_s")); + + // Measure rangeCost (the range clause's own match count) and derive the branch the same way + // IndexOrDocValuesQuery does: rangeCost >>> 3 <= leadCost -> points, else docValues. leadCost + // is + // the actual doc-frequency of the term this scenario leads with. + String window = criteria.equals("within") ? WITHIN_QUERY : OTHER_QUERY; + SolrQuery rangeOnly = + new SolrQuery( + "q", "{!numericRange criteria=" + criteria + " field=" + field + "}" + window); + rangeOnly.setParam("rows", "0"); + long rangeCost = + new QueryRequest(rangeOnly) + .processWithBaseUrl(solrBenchState.client, basePath, COLLECTION) + .getResults() + .getNumFound(); + long threshold = rangeCost >>> 3; + boolean hasDocValues = field.endsWith("_dv"); + long leadCost; + long dvBranch; // 1 = docValues verify branch, 0 = points/BKD + if (scenario.startsWith("none")) { + leadCost = 0; // standalone: no lead -> points + dvBranch = 0; + } else { + String facetField = scenario.startsWith("selective") ? "term_high_s" : "term_low_s"; + leadCost = firstFacetCount(response, facetField); + // IndexOrDocValuesQuery picks docValues only when the range is >8x the lead (threshold > + // leadCost); otherwise points. A docValues=false field never gets that choice. + dvBranch = (hasDocValues && threshold > leadCost) ? 1 : 0; + } + log( + "[range-bench] criteria=" + + criteria + + " field=" + + field + + " scenario=" + + scenario + + " rangeCost=" + + rangeCost + + " threshold=" + + threshold + + " leadCost=" + + leadCost + + " -> " + + (dvBranch == 1 ? "docValues" : "points")); + } + + private Set facetValues(QueryResponse response, String fieldName) { + return response.getFacetField(fieldName).getValues().stream() + .map(FacetField.Count::getName) + .collect(Collectors.toSet()); + } + + private long firstFacetCount(QueryResponse response, String fieldName) { + return response.getFacetField(fieldName).getValues().stream() + .mapToLong(FacetField.Count::getCount) + .findFirst() + .orElse(0L); + } + + @Setup(Level.Iteration) + public void setupIteration(SolrBenchState solrBenchState) + throws SolrServerException, IOException { + CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); + solrBenchState.client.requestWithBaseUrl(solrBenchState.nodes.getFirst(), reload, null); + } + + /** Builds the query for the current (criteria, field, scenario) parameter combination. */ + QueryRequest query() { + // within needs a window WIDER than the doc ranges (so they can fit inside it); the other + // relations need a narrow window that the doc ranges contain / overlap. + String window = criteria.equals("within") ? WITHIN_QUERY : OTHER_QUERY; + String rangeExpr = "{!numericRange criteria=" + criteria + " field=" + field + "}" + window; + if (scenario.startsWith("none")) { + return new QueryRequest(new SolrQuery("q", rangeExpr)); // standalone: no lead -> points + } + + boolean selective = scenario.startsWith("selective"); + Iterator terms = selective ? selectiveTerms : broadTerms; + String termField = selective ? "term_high_s" : "term_low_s"; + // Nest the {!numericRange} subquery via _query_ so it joins the term clause's BooleanQuery. + String query = "+" + termField + ":\"" + terms.next() + "\" +_query_:\"" + rangeExpr + "\""; + return new QueryRequest(new SolrQuery("q", query)); + } + } + + @Benchmark + public void rangeQuery(Blackhole blackhole, BenchState benchState, SolrBenchState solrBenchState) + throws SolrServerException, IOException { + // Sink the result exactly once, via Blackhole + blackhole.consume(benchState.query().process(solrBenchState.client, COLLECTION)); + } +} diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml index e3f77a826047..79b26fc9b3cd 100644 --- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml +++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml @@ -28,6 +28,7 @@ positionIncrementGap="0"/> + @@ -58,5 +59,11 @@ - id + + + + + + + id diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 1452233beee3..d57965d85b41 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -351,6 +351,36 @@ public List createFields(SchemaField field, Object value) { return f == null ? List.of() : List.of(f); } + /** + * Whether this type needs all values of a multiValued field together (rather than one value at a + * time) in order to build its indexable fields - for example to pack several ranges into a single + * {@code BinaryDocValues} blob. When {@code true}, {@link org.apache.solr.update.DocumentBuilder} + * calls {@link #createFieldsFromAllValues(SchemaField, Collection)} once per (multiValued) field + * instead of {@link #createFields(SchemaField, Object)} per value. + */ + public boolean createsFieldsFromAllValues() { + return false; + } + + /** + * Builds the indexable fields for all values of {@code field} in a single document at + * once. Only invoked when {@link #createsFieldsFromAllValues()} returns {@code true}; the default + * simply concatenates {@link #createFields(SchemaField, Object)} over each value, preserving + * per-value behavior. + * + * @param field the schema field + * @param values all (non-null) values for this field in the current document + * @return the indexable fields to add to the document + */ + public List createFieldsFromAllValues( + SchemaField field, Collection values) { + List fields = new ArrayList<>(); + for (Object value : values) { + fields.addAll(createFields(field, value)); + } + return fields; + } + /** * Convert an external value (from XML update command or from query string) into the internal * format for both storing and indexing (which can be modified by any analyzers). diff --git a/solr/core/src/java/org/apache/solr/schema/numericrange/AbstractNumericRangeField.java b/solr/core/src/java/org/apache/solr/schema/numericrange/AbstractNumericRangeField.java index a05a766ac52d..9bc3f6d8eea7 100644 --- a/solr/core/src/java/org/apache/solr/schema/numericrange/AbstractNumericRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/numericrange/AbstractNumericRangeField.java @@ -18,13 +18,20 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.RangeFieldQuery.QueryType; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.response.TextResponseWriter; @@ -134,13 +141,20 @@ protected Pattern getSingleBoundPattern() { @Override protected boolean enableDocValuesByDefault() { - return false; // Range fields do not support docValues + // DocValues are supported for both single and multiValued range fields, but are opt-in: they + // add an extra (binary) docValues field, and range docValues only help queries that combine + // the range clause with a more selective clause, so we don't enable them by default. + return false; } @Override protected void init(IndexSchema schema, Map args) { super.init(schema, args); + // Force useDocValuesAsStored off; it otherwise defaults on for schema, which + // would make fl=* responses fail on a docValues-enabled range field. + properties &= ~USE_DOCVALUES_AS_STORED; + String numDimensionsStr = args.remove("numDimensions"); if (numDimensionsStr != null) { numDimensions = Integer.parseInt(numDimensionsStr); @@ -153,27 +167,33 @@ protected void init(IndexSchema schema, Map args) { + typeName); } } + } - // Range fields do not support docValues - validate this wasn't explicitly enabled - if (hasProperty(DOC_VALUES)) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "docValues=true enabled but " - + getClass().getSimpleName() - + " does not support docValues for field type " - + typeName); - } + @Override + protected void checkSupportsDocValues() { + // DocValues are supported for both single and multiValued range fields (backed by binary + // docValues). } @Override public List createFields(SchemaField field, Object value) { - IndexableField indexedField = createField(field, value); List fields = new ArrayList<>(); + IndexableField indexedField = createField(field, value); if (indexedField != null) { fields.add(indexedField); } + if (field.hasDocValues() && !field.multiValued()) { + // Single-valued: one flat BinaryDocValues blob (read directly, no dictionary). multiValued + // docValues are built from all values at once in createFieldsFromAllValues (one blob holding + // every range), since BinaryDocValues holds only one value per document. + NumericRangeValue rv = parseRangeValue(value.toString()); + fields.add( + new BinaryDocValuesField( + field.getName(), BytesRef.deepCopyOf(encodePackedValue(field.getName(), rv)))); + } + if (field.stored()) { fields.add(getStoredField(field, value.toString())); } @@ -181,6 +201,86 @@ public List createFields(SchemaField field, Object value) { return fields; } + @Override + public boolean createsFieldsFromAllValues() { + // multiValued docValues range fields pack every range of a document into ONE BinaryDocValues + // blob (flat, no dictionary), so the type needs all values together to build it. + return true; + } + + @Override + public List createFieldsFromAllValues( + SchemaField field, Collection values) { + List fields = new ArrayList<>(); + // Indexed (BKD) and stored fields: one per value (same as the per-value path). + for (Object value : values) { + IndexableField indexedField = createField(field, value); + if (indexedField != null) { + fields.add(indexedField); + } + if (field.stored()) { + fields.add(getStoredField(field, value.toString())); + } + } + // docValues: a single BinaryDocValues value holding all of the document's ranges. + if (field.hasDocValues()) { + fields.add( + new BinaryDocValuesField(field.getName(), encodePackedValues(field.getName(), values))); + } + return fields; + } + + /** + * Encodes several ranges into one {@code BinaryDocValues} blob: each range's fixed-width {@code + * [min... | max...]} bytes concatenated. All ranges of a field share the same width, so the query + * recovers the count as {@code blob.length / stride}. + */ + protected BytesRef encodePackedValues(String field, Collection values) { + BytesRefBuilder builder = new BytesRefBuilder(); + for (Object value : values) { + builder.append(encodePackedValue(field, parseRangeValue(value.toString()))); + } + return builder.toBytesRef(); + } + + /** + * Encodes a range value into the packed {@code [min... | max...]} byte representation used by + * both the indexed docValues and the query. Reuses Lucene's own encoder (via the type-specific + * {@code *RangeDocValuesField}) so the docValues and BKD encodings stay identical. + */ + protected abstract BytesRef encodePackedValue(String field, NumericRangeValue rangeValue); + + /** Number of bytes per dimension value for this type (e.g. {@code Integer.BYTES}). */ + protected abstract int bytesPerDimension(); + + /** + * If the field has docValues, wraps the BKD query in an {@link IndexOrDocValuesQuery} whose + * docValues clause ({@link MultiBinaryRangeDocValuesQuery}) can cheaply verify candidates when a + * more selective clause leads iteration; otherwise returns the BKD query unchanged. It reads a + * flat per-doc blob of one (single-valued) or several (multiValued) ranges, avoiding the + * dictionary/ordinal overhead of SortedSet docValues. + */ + protected Query maybeWrapWithDocValues( + SchemaField field, QueryType type, NumericRangeValue rangeValue, Query bkdQuery) { + if (!field.hasDocValues()) { + return bkdQuery; + } + BytesRef packed = encodePackedValue(field.getName(), rangeValue); + byte[] queryPackedValue = + Arrays.copyOfRange(packed.bytes, packed.offset, packed.offset + packed.length); + Query dv = + new MultiBinaryRangeDocValuesQuery( + field.getName(), + queryPackedValue, + rangeValue.getDimensions(), + bytesPerDimension(), + type); + if (!field.indexed()) { + return dv; + } + return new IndexOrDocValuesQuery(bkdQuery, dv); + } + protected StoredField getStoredField(SchemaField sf, Object value) { return new StoredField(sf.getName(), value.toString()); } @@ -278,7 +378,7 @@ public Object toNativeType(Object val) { * @param rangeValue a pre-parsed range value produced by this field type * @return a contains query for the given field and range */ - public abstract Query newContainsQuery(String field, NumericRangeValue rangeValue); + public abstract Query newContainsQuery(SchemaField field, NumericRangeValue rangeValue); /** * Creates a Lucene query that matches indexed documents whose stored range intersects @@ -288,7 +388,7 @@ public Object toNativeType(Object val) { * @param rangeValue a pre-parsed range value produced by this field type * @return an intersects query for the given field and range */ - public abstract Query newIntersectsQuery(String field, NumericRangeValue rangeValue); + public abstract Query newIntersectsQuery(SchemaField field, NumericRangeValue rangeValue); /** * Creates a Lucene query that matches indexed documents whose stored range is within the @@ -298,7 +398,7 @@ public Object toNativeType(Object val) { * @param rangeValue a pre-parsed range value produced by this field type * @return a within query for the given field and range */ - public abstract Query newWithinQuery(String field, NumericRangeValue rangeValue); + public abstract Query newWithinQuery(SchemaField field, NumericRangeValue rangeValue); /** * Creates a Lucene query that matches indexed documents whose stored range crosses the @@ -308,7 +408,7 @@ public Object toNativeType(Object val) { * @param rangeValue a pre-parsed range value produced by this field type * @return a crosses query for the given field and range */ - public abstract Query newCrossesQuery(String field, NumericRangeValue rangeValue); + public abstract Query newCrossesQuery(SchemaField field, NumericRangeValue rangeValue); /** * Creates a query for this field that matches docs where the query-range is fully contained by @@ -336,7 +436,7 @@ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal // Check if it's the full range syntax: [min1,min2 TO max1,max2] if (getRangePattern().matcher(trimmed).matches()) { final var rangeValue = parseRangeValue(trimmed); - return newContainsQuery(field.getName(), rangeValue); + return newContainsQuery(field, rangeValue); } // Syntax sugar: also accept a single-bound (i.e pX,pY,pZ) @@ -353,7 +453,10 @@ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal + ")"); } - return newContainsQuery(field.getName(), singleBoundRange); + // A single bound is the degenerate range [p,p]; "contains p" is equivalent to + // "intersects [p,p]", so route it through the intersects path to pick up the docValues + // optimization for point queries. + return newIntersectsQuery(field, singleBoundRange); } throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/schema/numericrange/DoubleRangeField.java b/solr/core/src/java/org/apache/solr/schema/numericrange/DoubleRangeField.java index c7eaeb5a82b8..5b98d7139fb6 100644 --- a/solr/core/src/java/org/apache/solr/schema/numericrange/DoubleRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/numericrange/DoubleRangeField.java @@ -19,8 +19,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.lucene.document.DoubleRange; +import org.apache.lucene.document.DoubleRangeDocValuesField; +import org.apache.lucene.document.RangeFieldQuery.QueryType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.schema.SchemaField; @@ -101,6 +104,17 @@ public IndexableField createField(SchemaField field, Object value) { return new DoubleRange(field.getName(), rangeValue.mins, rangeValue.maxs); } + @Override + protected BytesRef encodePackedValue(String field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return new DoubleRangeDocValuesField(field, rv.mins, rv.maxs).binaryValue(); + } + + @Override + protected int bytesPerDimension() { + return Double.BYTES; + } + /** * Parse a range value string into a RangeValue object. * @@ -196,27 +210,43 @@ private double[] parseDoubleArray(String str, String description) { } @Override - public Query newContainsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return DoubleRange.newContainsQuery(fieldName, rv.mins, rv.maxs); + public Query newContainsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CONTAINS, + rangeValue, + DoubleRange.newContainsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newIntersectsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return DoubleRange.newIntersectsQuery(fieldName, rv.mins, rv.maxs); + public Query newIntersectsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.INTERSECTS, + rangeValue, + DoubleRange.newIntersectsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newWithinQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return DoubleRange.newWithinQuery(fieldName, rv.mins, rv.maxs); + public Query newWithinQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.WITHIN, + rangeValue, + DoubleRange.newWithinQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newCrossesQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return DoubleRange.newCrossesQuery(fieldName, rv.mins, rv.maxs); + public Query newCrossesQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CROSSES, + rangeValue, + DoubleRange.newCrossesQuery(field.getName(), rv.mins, rv.maxs)); } @Override @@ -261,7 +291,9 @@ protected Query getSpecializedRangeQuery( if (numDimensions == 1) { mins[0] = min; maxs[0] = max; - return DoubleRange.newContainsQuery(field.getName(), mins, maxs); + // Route through newContainsQuery so the standard field:[X TO Y] syntax also picks up the + // docValues optimization when the field has docValues. + return newContainsQuery(field, new RangeValue(mins, maxs)); } else { throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/schema/numericrange/FloatRangeField.java b/solr/core/src/java/org/apache/solr/schema/numericrange/FloatRangeField.java index 0d4e4f31d583..edae9f93dd95 100644 --- a/solr/core/src/java/org/apache/solr/schema/numericrange/FloatRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/numericrange/FloatRangeField.java @@ -19,8 +19,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.lucene.document.FloatRange; +import org.apache.lucene.document.FloatRangeDocValuesField; +import org.apache.lucene.document.RangeFieldQuery.QueryType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.schema.SchemaField; @@ -101,6 +104,17 @@ public IndexableField createField(SchemaField field, Object value) { return new FloatRange(field.getName(), rangeValue.mins, rangeValue.maxs); } + @Override + protected BytesRef encodePackedValue(String field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return new FloatRangeDocValuesField(field, rv.mins, rv.maxs).binaryValue(); + } + + @Override + protected int bytesPerDimension() { + return Float.BYTES; + } + /** * Parse a range value string into a RangeValue object. * @@ -196,27 +210,43 @@ private float[] parseFloatArray(String str, String description) { } @Override - public Query newContainsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return FloatRange.newContainsQuery(fieldName, rv.mins, rv.maxs); + public Query newContainsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CONTAINS, + rangeValue, + FloatRange.newContainsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newIntersectsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return FloatRange.newIntersectsQuery(fieldName, rv.mins, rv.maxs); + public Query newIntersectsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.INTERSECTS, + rangeValue, + FloatRange.newIntersectsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newWithinQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return FloatRange.newWithinQuery(fieldName, rv.mins, rv.maxs); + public Query newWithinQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.WITHIN, + rangeValue, + FloatRange.newWithinQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newCrossesQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return FloatRange.newCrossesQuery(fieldName, rv.mins, rv.maxs); + public Query newCrossesQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CROSSES, + rangeValue, + FloatRange.newCrossesQuery(field.getName(), rv.mins, rv.maxs)); } @Override @@ -261,7 +291,9 @@ protected Query getSpecializedRangeQuery( if (numDimensions == 1) { mins[0] = min; maxs[0] = max; - return FloatRange.newContainsQuery(field.getName(), mins, maxs); + // Route through newContainsQuery so the standard field:[X TO Y] syntax also picks up the + // docValues optimization when the field has docValues. + return newContainsQuery(field, new RangeValue(mins, maxs)); } else { throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/schema/numericrange/IntRangeField.java b/solr/core/src/java/org/apache/solr/schema/numericrange/IntRangeField.java index b8e76d0a680b..b2faec3534e9 100644 --- a/solr/core/src/java/org/apache/solr/schema/numericrange/IntRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/numericrange/IntRangeField.java @@ -18,8 +18,11 @@ import java.util.regex.Matcher; import org.apache.lucene.document.IntRange; +import org.apache.lucene.document.IntRangeDocValuesField; +import org.apache.lucene.document.RangeFieldQuery.QueryType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.schema.SchemaField; @@ -89,6 +92,17 @@ public IndexableField createField(SchemaField field, Object value) { return new IntRange(field.getName(), rangeValue.mins, rangeValue.maxs); } + @Override + protected BytesRef encodePackedValue(String field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return new IntRangeDocValuesField(field, rv.mins, rv.maxs).binaryValue(); + } + + @Override + protected int bytesPerDimension() { + return Integer.BYTES; + } + /** * Parse a range value string into a RangeValue object. * @@ -184,27 +198,43 @@ private int[] parseIntArray(String str, String description) { } @Override - public Query newContainsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rangeValueTyped = (RangeValue) rangeValue; - return IntRange.newContainsQuery(fieldName, rangeValueTyped.mins, rangeValueTyped.maxs); + public Query newContainsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CONTAINS, + rangeValue, + IntRange.newContainsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newIntersectsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return IntRange.newIntersectsQuery(fieldName, rv.mins, rv.maxs); + public Query newIntersectsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.INTERSECTS, + rangeValue, + IntRange.newIntersectsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newWithinQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return IntRange.newWithinQuery(fieldName, rv.mins, rv.maxs); + public Query newWithinQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.WITHIN, + rangeValue, + IntRange.newWithinQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newCrossesQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return IntRange.newCrossesQuery(fieldName, rv.mins, rv.maxs); + public Query newCrossesQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CROSSES, + rangeValue, + IntRange.newCrossesQuery(field.getName(), rv.mins, rv.maxs)); } @Override @@ -248,7 +278,9 @@ protected Query getSpecializedRangeQuery( if (numDimensions == 1) { mins[0] = min; maxs[0] = max; - return IntRange.newContainsQuery(field.getName(), mins, maxs); + // Route through newContainsQuery so the standard field:[X TO Y] syntax also picks up the + // docValues optimization when the field has docValues. + return newContainsQuery(field, new RangeValue(mins, maxs)); } else { throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/schema/numericrange/LongRangeField.java b/solr/core/src/java/org/apache/solr/schema/numericrange/LongRangeField.java index 5004f3affb31..6057a9f81ef0 100644 --- a/solr/core/src/java/org/apache/solr/schema/numericrange/LongRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/numericrange/LongRangeField.java @@ -18,8 +18,11 @@ import java.util.regex.Matcher; import org.apache.lucene.document.LongRange; +import org.apache.lucene.document.LongRangeDocValuesField; +import org.apache.lucene.document.RangeFieldQuery.QueryType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.schema.SchemaField; @@ -90,6 +93,17 @@ public IndexableField createField(SchemaField field, Object value) { return new LongRange(field.getName(), rangeValue.mins, rangeValue.maxs); } + @Override + protected BytesRef encodePackedValue(String field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return new LongRangeDocValuesField(field, rv.mins, rv.maxs).binaryValue(); + } + + @Override + protected int bytesPerDimension() { + return Long.BYTES; + } + /** * Parse a range value string into a RangeValue object. * @@ -185,27 +199,43 @@ private long[] parseLongArray(String str, String description) { } @Override - public Query newContainsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rangeValueTyped = (RangeValue) rangeValue; - return LongRange.newContainsQuery(fieldName, rangeValueTyped.mins, rangeValueTyped.maxs); + public Query newContainsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CONTAINS, + rangeValue, + LongRange.newContainsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newIntersectsQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return LongRange.newIntersectsQuery(fieldName, rv.mins, rv.maxs); + public Query newIntersectsQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.INTERSECTS, + rangeValue, + LongRange.newIntersectsQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newWithinQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return LongRange.newWithinQuery(fieldName, rv.mins, rv.maxs); + public Query newWithinQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.WITHIN, + rangeValue, + LongRange.newWithinQuery(field.getName(), rv.mins, rv.maxs)); } @Override - public Query newCrossesQuery(String fieldName, NumericRangeValue rangeValue) { - final var rv = (RangeValue) rangeValue; - return LongRange.newCrossesQuery(fieldName, rv.mins, rv.maxs); + public Query newCrossesQuery(SchemaField field, NumericRangeValue rangeValue) { + RangeValue rv = (RangeValue) rangeValue; + return maybeWrapWithDocValues( + field, + QueryType.CROSSES, + rangeValue, + LongRange.newCrossesQuery(field.getName(), rv.mins, rv.maxs)); } @Override @@ -249,7 +279,9 @@ protected Query getSpecializedRangeQuery( if (numDimensions == 1) { mins[0] = min; maxs[0] = max; - return LongRange.newContainsQuery(field.getName(), mins, maxs); + // Route through newContainsQuery so the standard field:[X TO Y] syntax also picks up the + // docValues optimization when the field has docValues. + return newContainsQuery(field, new RangeValue(mins, maxs)); } else { throw new SolrException( ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/schema/numericrange/MultiBinaryRangeDocValuesQuery.java b/solr/core/src/java/org/apache/solr/schema/numericrange/MultiBinaryRangeDocValuesQuery.java new file mode 100644 index 000000000000..f8404ebeba39 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/numericrange/MultiBinaryRangeDocValuesQuery.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema.numericrange; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import org.apache.lucene.document.RangeFieldQuery; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.ArrayUtil.ByteArrayComparator; +import org.apache.lucene.util.BytesRef; + +/** + * A docValues-backed range query over {@link BinaryDocValues} for numeric range field types + * (single- or multiValued). + * + *

A document's range(s) are packed into a single {@code BinaryDocValues} value as one or more + * concatenated fixed-width {@code [min... | max...]} blobs (see {@code + * AbstractNumericRangeField.encodePackedValues}). Because every range for a field is the same width + * ({@code stride = 2 * numDims * bytesPerDim}), the count is recovered as {@code value.length / + * stride}. + * + *

Reading the flat per-doc blob avoids the dictionary/ordinal dereference a SortedSet-backed + * docValues range query would pay, so it stays cheap even for high-cardinality range data. It is a + * candidate to push down to Lucene, which currently lacks a multi-valued binary range docValues + * field. + */ +final class MultiBinaryRangeDocValuesQuery extends Query { + + private final String field; + private final byte[] queryPackedValue; + private final int numDims; + private final int bytesPerDim; + private final RangeFieldQuery.QueryType queryType; + private final ByteArrayComparator comparator; + + /** + * @param field the field name + * @param queryPackedValue the query range, encoded as {@code [min... | max...]} the same way the + * indexed ranges are + * @param numDims number of dimensions (1-4) + * @param bytesPerDim bytes per dimension value (e.g. {@code Integer.BYTES}) + * @param queryType the relation to test ({@code INTERSECTS}, {@code CONTAINS}, {@code WITHIN}, + * {@code CROSSES}) + */ + MultiBinaryRangeDocValuesQuery( + String field, + byte[] queryPackedValue, + int numDims, + int bytesPerDim, + RangeFieldQuery.QueryType queryType) { + this.field = Objects.requireNonNull(field, "field must not be null"); + this.queryPackedValue = + Objects.requireNonNull(queryPackedValue, "queryPackedValue must not be null"); + this.numDims = numDims; + this.bytesPerDim = bytesPerDim; + this.queryType = Objects.requireNonNull(queryType, "queryType must not be null"); + this.comparator = ArrayUtil.getUnsignedComparator(bytesPerDim); + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + LeafReader reader = context.reader(); + if (reader.getFieldInfos().fieldInfo(field) == null) { + return null; + } + final BinaryDocValues values = DocValues.getBinary(reader, field); + final int stride = 2 * numDims * bytesPerDim; + final byte[] docPacked = new byte[stride]; + final TwoPhaseIterator iterator = + new TwoPhaseIterator(values) { + @Override + public boolean matches() throws IOException { + // The doc's value is N concatenated stride-byte ranges; match if ANY satisfies. + // Copy each range to a 0-based buffer because QueryType.matches indexes from 0. + BytesRef b = values.binaryValue(); + for (int off = b.offset; off < b.offset + b.length; off += stride) { + System.arraycopy(b.bytes, off, docPacked, 0, stride); + if (queryType.matches( + queryPackedValue, docPacked, numDims, bytesPerDim, comparator)) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + return stride; // ~ one packed-value comparison per range + } + }; + + Scorer scorer = new ConstantScoreScorer(score(), scoreMode, iterator); + return new DefaultScorerSupplier(scorer); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return DocValues.isCacheable(ctx, field); + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } + + @Override + public String toString(String f) { + return getClass().getSimpleName() + "(field=" + field + ", type=" + queryType + ")"; + } + + @Override + public boolean equals(Object obj) { + if (!sameClassAs(obj)) { + return false; + } + MultiBinaryRangeDocValuesQuery that = (MultiBinaryRangeDocValuesQuery) obj; + // queryType MUST be part of identity: the same field/range under different relations are + // different queries and must not collide in the query cache. + return numDims == that.numDims + && bytesPerDim == that.bytesPerDim + && queryType == that.queryType + && field.equals(that.field) + && Arrays.equals(queryPackedValue, that.queryPackedValue); + } + + @Override + public int hashCode() { + int h = classHash(); + h = 31 * h + field.hashCode(); + h = 31 * h + Arrays.hashCode(queryPackedValue); + h = 31 * h + numDims; + h = 31 * h + bytesPerDim; + h = 31 * h + queryType.hashCode(); + return h; + } +} diff --git a/solr/core/src/java/org/apache/solr/search/numericrange/NumericRangeQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/numericrange/NumericRangeQParserPlugin.java index 94e113d805e4..fce0cb8ee6dc 100644 --- a/solr/core/src/java/org/apache/solr/search/numericrange/NumericRangeQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/numericrange/NumericRangeQParserPlugin.java @@ -194,10 +194,10 @@ public Query parse() throws SyntaxError { throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid range value: " + rangeValue, e); } return switch (criteria) { - case INTERSECTS -> rangeField.newIntersectsQuery(fieldName, range); - case WITHIN -> rangeField.newWithinQuery(fieldName, range); - case CONTAINS -> rangeField.newContainsQuery(fieldName, range); - case CROSSES -> rangeField.newCrossesQuery(fieldName, range); + case INTERSECTS -> rangeField.newIntersectsQuery(schemaField, range); + case WITHIN -> rangeField.newWithinQuery(schemaField, range); + case CONTAINS -> rangeField.newContainsQuery(schemaField, range); + case CROSSES -> rangeField.newCrossesQuery(schemaField, range); }; } else { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java index 0a13b967989d..f2ec9bfc2dad 100644 --- a/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java +++ b/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java @@ -207,6 +207,42 @@ public static Document toDocument( usedFields); } } + } else if (sfield != null + && sfield.multiValued() + && !forInPlaceUpdate + && sfield.getType().createsFieldsFromAllValues()) { + // Types that must combine a field's values into a single indexable field (e.g. all of a + // doc's ranges into one BinaryDocValues blob) receive them together. copyFields stay + // per-value. + List values = new ArrayList<>(field.getValueCount()); + for (Object v : field) { + if (v != null) { + values.add(v); + } + } + if (!values.isEmpty()) { + hasField = true; + for (IndexableField f : sfield.getType().createFieldsFromAllValues(sfield, values)) { + if (f != null) { + out.add(f); + } + } + usedFields.add(sfield.getName()); + used = true; + if (copyFields != null) { + for (Object v : values) { + addCopyFields( + schema, + v, + sfield.getType(), + copyFields, + forInPlaceUpdate, + uniqueKeyFieldName, + out, + usedFields); + } + } + } } else { Iterator it = field.iterator(); while (it.hasNext()) { diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-numericrange.xml b/solr/core/src/test-files/solr/collection1/conf/schema-numericrange.xml index 660acc1bfb1a..dcab0099f63b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-numericrange.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-numericrange.xml @@ -105,6 +105,24 @@ + + + + + + + + + + + + + + + + + + id diff --git a/solr/core/src/test/org/apache/solr/schema/numericrange/MultiBinaryRangeDocValuesQueryTest.java b/solr/core/src/test/org/apache/solr/schema/numericrange/MultiBinaryRangeDocValuesQueryTest.java new file mode 100644 index 000000000000..1af2f79fa745 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/numericrange/MultiBinaryRangeDocValuesQueryTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema.numericrange; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.IntRange; +import org.apache.lucene.document.IntRangeDocValuesField; +import org.apache.lucene.document.RangeFieldQuery.QueryType; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.CollectorManager; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.FixedBitSet; +import org.apache.solr.SolrTestCase; +import org.junit.Test; + +/** + * Unit tests for {@link MultiBinaryRangeDocValuesQuery}, the binary docValues-backed range + * verifier. + * + *

Each doc indexes its range(s) both as BKD ({@link IntRange}) and as a single {@code + * BinaryDocValues} blob (all ranges concatenated, as {@code AbstractNumericRangeField} does), + * letting us assert the docValues verifier returns exactly the same documents as the trusted BKD + * query across all four relations - for both single-valued docs (one range) and multiValued docs + * (several ranges in one blob), where "any range matches" (not the union of ranges) is correct. + */ +public class MultiBinaryRangeDocValuesQueryTest extends SolrTestCase { + + private static final String FIELD = "r"; + + /** Encodes a 1-D query range the same way indexed ranges are encoded. */ + private static byte[] encode(int min, int max) { + BytesRef b = new IntRangeDocValuesField(FIELD, new int[] {min}, new int[] {max}).binaryValue(); + return Arrays.copyOfRange(b.bytes, b.offset, b.offset + b.length); + } + + private static Query dv(QueryType type, int min, int max) { + return new MultiBinaryRangeDocValuesQuery(FIELD, encode(min, max), 1, IntRange.BYTES, type); + } + + private static Query bkd(QueryType type, int min, int max) { + int[] mn = {min}; + int[] mx = {max}; + return switch (type) { + case INTERSECTS -> IntRange.newIntersectsQuery(FIELD, mn, mx); + case CONTAINS -> IntRange.newContainsQuery(FIELD, mn, mx); + case WITHIN -> IntRange.newWithinQuery(FIELD, mn, mx); + case CROSSES -> IntRange.newCrossesQuery(FIELD, mn, mx); + }; + } + + /** + * Adds a document whose (possibly multiple) ranges are indexed as BKD and as one BinaryDocValues + * blob holding every range concatenated (a single-range doc is just the one-range case). + */ + private static void addDoc(RandomIndexWriter w, int[]... ranges) throws IOException { + Document doc = new Document(); + BytesRefBuilder blob = new BytesRefBuilder(); + for (int[] r : ranges) { + int[] mn = {r[0]}; + int[] mx = {r[1]}; + doc.add(new IntRange(FIELD, mn, mx)); // BKD (ground truth) + blob.append(new IntRangeDocValuesField(FIELD, mn, mx).binaryValue()); // concat into one blob + } + doc.add(new BinaryDocValuesField(FIELD, blob.toBytesRef())); + w.addDocument(doc); + } + + private static FixedBitSet matches(IndexSearcher s, Query q) throws IOException { + int maxDoc = Math.max(1, s.getIndexReader().maxDoc()); + return s.search( + q, + new CollectorManager() { + @Override + public BitSetCollector newCollector() { + return new BitSetCollector(maxDoc); + } + + @Override + public FixedBitSet reduce(Collection collectors) { + FixedBitSet merged = new FixedBitSet(maxDoc); + for (BitSetCollector c : collectors) { + merged.or(c.bits); + } + return merged; + } + }); + } + + /** + * Collects matching doc ids (segment-local -> absolute) into a bitset, per CollectorManager. + */ + private static final class BitSetCollector extends SimpleCollector { + final FixedBitSet bits; + int base; + + BitSetCollector(int maxDoc) { + this.bits = new FixedBitSet(maxDoc); + } + + @Override + public void collect(int doc) { + bits.set(base + doc); + } + + @Override + protected void doSetNextReader(LeafReaderContext ctx) { + base = ctx.docBase; + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + } + + @Test + public void testMultiValuedPerRangeSemantics() throws Exception { + try (Directory dir = newDirectory()) { + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + addDoc(w, new int[] {1, 10}, new int[] {5, 20}); // doc: two overlapping ranges + addDoc(w, new int[] {0, 5}, new int[] {100, 200}); // doc: two disjoint ranges + addDoc(w, new int[] {130, 160}); // doc: single range + try (IndexReader reader = w.getReader()) { + w.close(); + IndexSearcher searcher = newSearcher(reader); + + // CONTAINS [1,15]: neither individual range of the first doc contains it; the *union* + // [1,20] would, but ranges are never unioned -> zero matches. + assertEquals(0, searcher.count(dv(QueryType.CONTAINS, 1, 15))); + assertEquals(0, searcher.count(bkd(QueryType.CONTAINS, 1, 15))); + + // CONTAINS [3,150]: the second doc's [0,5] and [100,200] each satisfy one half-open bound, + // so a decomposition would falsely match -- native per-range matching must not. + assertEquals(0, searcher.count(dv(QueryType.CONTAINS, 3, 150))); + + // CONTAINS [1,10]: first doc's [1,10] contains it (any range suffices). + assertEquals(1, searcher.count(dv(QueryType.CONTAINS, 1, 10))); + + // INTERSECTS [12,14]: first doc's [5,20] overlaps it. + assertEquals(1, searcher.count(dv(QueryType.INTERSECTS, 12, 14))); + + // WITHIN [0,25]: first doc ([1,10]) and second doc ([0,5]) each have a range within it. + assertEquals(2, searcher.count(dv(QueryType.WITHIN, 0, 25))); + + // Every case must agree with the BKD ground truth. + for (QueryType t : QueryType.values()) { + for (int[] q : new int[][] {{1, 15}, {3, 150}, {1, 10}, {12, 14}, {0, 25}}) { + assertEquals( + "relation " + t + " query [" + q[0] + " TO " + q[1] + "]", + matches(searcher, bkd(t, q[0], q[1])), + matches(searcher, dv(t, q[0], q[1]))); + } + } + } + } + } + + @Test + public void testParityWithBkdRandom() throws Exception { + try (Directory dir = newDirectory()) { + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + int numDocs = atLeast(200); + for (int i = 0; i < numDocs; i++) { + int numRanges = TestUtil.nextInt(random(), 1, 3); // single- and multi-valued docs + int[][] ranges = new int[numRanges][2]; + for (int r = 0; r < numRanges; r++) { + int a = TestUtil.nextInt(random(), -50, 50); + int b = TestUtil.nextInt(random(), a, 50); // a <= b + ranges[r] = new int[] {a, b}; + } + addDoc(w, ranges); + } + try (IndexReader reader = w.getReader()) { + w.close(); + IndexSearcher searcher = newSearcher(reader); + + for (int iter = 0; iter < 200; iter++) { + int qa = TestUtil.nextInt(random(), -55, 55); + int qb = TestUtil.nextInt(random(), qa, 55); + for (QueryType t : QueryType.values()) { + assertEquals( + "relation " + t + " query [" + qa + " TO " + qb + "]", + matches(searcher, bkd(t, qa, qb)), + matches(searcher, dv(t, qa, qb))); + } + } + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/search/numericrange/NumericRangeDocValuesTest.java b/solr/core/src/test/org/apache/solr/search/numericrange/NumericRangeDocValuesTest.java new file mode 100644 index 000000000000..784c842be208 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/numericrange/NumericRangeDocValuesTest.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.numericrange; + +import java.util.List; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests over the {@code schema-numericrange.xml} verifying that the docValues-enabled + * range fields return exactly the same results as their indexed-only counterparts. + * + *

The docValues-enabled fields wrap their BKD query in a Lucene {@link + * org.apache.lucene.search.IndexOrDocValuesQuery}. These tests assert that this produces identical + * matches to the direct BKD query across intersects / contains / single-bound / multi-dimensional + * queries, inside a selective conjunction, and for the docValues-only (non-indexed) field. + */ +public class NumericRangeDocValuesTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema-numericrange.xml"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + clearIndex(); + assertU(commit()); + } + + /** + * Index the same int range into the indexed-only, indexed+docValues, and docValues-only fields. + */ + private void addIntDoc(String id, String range) { + assertU( + adoc( + "id", id, + "price_range", range, + "price_range_dv", range, + "price_range_dvonly", range)); + } + + private List get1dIntRangeFieldNames() { + return List.of("price_range", "price_range_dv", "price_range_dvonly"); + } + + @Test + public void testIntersectsParity() { + addIntDoc("1", "[100 TO 200]"); + addIntDoc("2", "[150 TO 250]"); + addIntDoc("3", "[50 TO 80]"); + assertU(commit()); + + // intersects [120 TO 180] -> docs 1 and 2, for every field variant + for (String field : get1dIntRangeFieldNames()) { + assertQ( + "intersects on " + field, + req("q", "{!numericRange criteria=intersects field=" + field + "}[120 TO 180]"), + "//result[@numFound='2']", + "//result/doc/str[@name='id'][.='1']", + "//result/doc/str[@name='id'][.='2']"); + } + } + + @Test + public void testContainsParity() { + addIntDoc("1", "[100 TO 200]"); // fully contains [120 TO 180] + addIntDoc("2", "[130 TO 160]"); // within, does NOT contain + addIntDoc("3", "[150 TO 250]"); // partial overlap, does NOT contain + assertU(commit()); + + for (String field : get1dIntRangeFieldNames()) { + assertQ( + "contains on " + field, + req("q", "{!numericRange criteria=contains field=" + field + "}[120 TO 180]"), + "//result[@numFound='1']", + "//result/doc/str[@name='id'][.='1']"); + } + } + + @Test + public void testStandardSyntaxContainsParity() { + addIntDoc("1", "[100 TO 200]"); // contains [120 TO 180] + addIntDoc("2", "[130 TO 160]"); // no + assertU(commit()); + + for (String field : get1dIntRangeFieldNames()) { + assertQ( + "standard-syntax contains on " + field, + req("q", field + ":[120 TO 180]"), + "//result[@numFound='1']", + "//result/doc/str[@name='id'][.='1']"); + } + } + + @Test + public void testSingleBoundParity() { + // A single bound (field:150) is a degenerate range [150,150]; contains == intersects for a + // point. + addIntDoc("1", "[100 TO 200]"); // contains 150 + addIntDoc("2", "[150 TO 150]"); // contains 150 + addIntDoc("3", "[100 TO 149]"); // no (max below 150) + addIntDoc("4", "[151 TO 300]"); // no (min above 150) + assertU(commit()); + + for (String f : get1dIntRangeFieldNames()) { + assertQ( + "single-bound on " + f, + req("q", f + ":150"), + "//result[@numFound='2']", + "//result/doc/str[@name='id'][.='1']", + "//result/doc/str[@name='id'][.='2']"); + } + } + + @Test + public void test2DContainsParity() { + assertU(adoc("id", "1", "bbox", "[0,0 TO 10,10]", "bbox_dv", "[0,0 TO 10,10]")); // contains + assertU(adoc("id", "2", "bbox", "[4,4 TO 6,6]", "bbox_dv", "[4,4 TO 6,6]")); // does not contain + assertU(commit()); + + for (String f : new String[] {"bbox", "bbox_dv"}) { + assertQ( + "2D contains on " + f, + req("q", "{!numericRange criteria=contains field=" + f + "}[3,3 TO 7,7]"), + "//result[@numFound='1']", + "//result/doc/str[@name='id'][.='1']"); + } + } + + @Test + public void testSelectiveConjunction() { + // Exercises IndexOrDocValuesQuery inside a conjunction: the selective title clause leads and + // the docValues range clause verifies. Must match the equivalent non-docValues query exactly. + assertU( + adoc( + "id", + "1", + "title", + "match", + "price_range", + "[100 TO 200]", + "price_range_dv", + "[100 TO 200]")); + assertU( + adoc( + "id", + "2", + "title", + "match", + "price_range", + "[300 TO 400]", + "price_range_dv", + "[300 TO 400]")); + assertU( + adoc( + "id", + "3", + "title", + "other", + "price_range", + "[100 TO 200]", + "price_range_dv", + "[100 TO 200]")); + assertU(commit()); + + for (String f : new String[] {"price_range", "price_range_dv"}) { + assertQ( + "selective AND contains on " + f, + req("q", "+title:match +" + f + ":[120 TO 180]"), + "//result[@numFound='1']", + "//result/doc/str[@name='id'][.='1']"); + } + } + + @Test + public void testLongParity() { + assertU(adoc("id", "1", "long_range", "[100 TO 200]", "long_range_dv", "[100 TO 200]")); + assertU(adoc("id", "2", "long_range", "[150 TO 250]", "long_range_dv", "[150 TO 250]")); + assertU(adoc("id", "3", "long_range", "[50 TO 80]", "long_range_dv", "[50 TO 80]")); + assertU(commit()); + + for (String f : new String[] {"long_range", "long_range_dv"}) { + assertQ( + "long intersects on " + f, + req("q", "{!numericRange criteria=intersects field=" + f + "}[120 TO 180]"), + "//result[@numFound='2']"); + } + } + + @Test + public void testFloatIntersectsParity() { + assertU(adoc("id", "1", "float_range", "[1.0 TO 2.0]", "float_range_dv", "[1.0 TO 2.0]")); + assertU(adoc("id", "2", "float_range", "[1.5 TO 3.0]", "float_range_dv", "[1.5 TO 3.0]")); + assertU(adoc("id", "3", "float_range", "[5.0 TO 6.0]", "float_range_dv", "[5.0 TO 6.0]")); + assertU(commit()); + + for (String f : new String[] {"float_range", "float_range_dv"}) { + assertQ( + "float intersects on " + f, + req("q", "{!numericRange criteria=intersects field=" + f + "}[1.2 TO 1.8]"), + "//result[@numFound='2']"); + } + } + + @Test + public void testDoubleContainsParity() { + // doc 1 fully contains [2.0 TO 4.0]; doc 2 is within it (does not contain) + assertU(adoc("id", "1", "double_range", "[1.0 TO 5.0]", "double_range_dv", "[1.0 TO 5.0]")); + assertU(adoc("id", "2", "double_range", "[2.5 TO 3.5]", "double_range_dv", "[2.5 TO 3.5]")); + assertU(commit()); + + for (String f : new String[] {"double_range", "double_range_dv"}) { + assertQ( + "double contains on " + f, + req("q", "{!numericRange criteria=contains field=" + f + "}[2.0 TO 4.0]"), + "//result[@numFound='1']", + "//result/doc/str[@name='id'][.='1']"); + } + } + + @Test + public void testSortingUnsupportedEvenWithDocValues() { + addIntDoc("1", "[100 TO 200]"); + assertU(commit()); + // Range fields don't implement sorting: there's no canonical key for ordering intervals (by + // min? max? width? midpoint?), multi-dimensional ranges (bbox/cube/tesseract) have no sensible + // order at all, and the packed BINARY docValues isn't a sortable scalar. Enabling docValues + // does not change that, getSortField still throws. + assertQEx( + "sorting on a range field must fail even with docValues", + "Cannot sort on", + req("q", "*:*", "sort", "price_range_dv asc"), + SolrException.ErrorCode.BAD_REQUEST); + } + + @Test + public void testQueryFacetingWithDocValues() { + // Range fields support facet.query (count docs matching a range query), the realistic way to + // facet ranges in Solr. (facet.field / value faceting is NOT supported: range docValues are + // BINARY, not the SortedSet docValues that term-faceting needs) + addIntDoc("1", "[100 TO 200]"); + addIntDoc("2", "[150 TO 250]"); + addIntDoc("3", "[50 TO 80]"); + assertU(commit()); + + assertQ( + req( + "q", "*:*", + "facet", "true", + "facet.query", + "{!numericRange criteria=intersects field=price_range_dv key=pr}[120 TO 180]"), + "//lst[@name='facet_queries']/int[@name='pr'][.='2']"); + } +} diff --git a/solr/core/src/test/org/apache/solr/search/numericrange/NumericRangeMultiValuedDocValuesTest.java b/solr/core/src/test/org/apache/solr/search/numericrange/NumericRangeMultiValuedDocValuesTest.java new file mode 100644 index 000000000000..7222318c5808 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/numericrange/NumericRangeMultiValuedDocValuesTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.numericrange; + +import java.util.ArrayList; +import java.util.List; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for multiValued range fields with docValues (backed by SortedSet docValues). + * + *

A document may hold several ranges. The docValues verifier ({@code + * MultiBinaryRangeDocValuesQuery}) matches a document when any of its ranges satisfies the + * relation, the same "any value matches" semantics as multiValued Points (BKD). Each test indexes + * the same ranges into a multiValued BKD-only field ({@code *_multi}) and a multiValued docValues + * field ({@code *_mv_dv}) and asserts they return identical counts, so the docValues path is + * checked against the trusted BKD path. + * + *

That these fields load at all also verifies that {@code multiValued=true docValues=true} is + * now accepted for range field types. + */ +public class NumericRangeMultiValuedDocValuesTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema-numericrange.xml"); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + clearIndex(); + assertU(commit()); + } + + /** Indexes the given ranges into both a BKD-only and a docValues multiValued field. */ + private void addMvDoc(String id, String bkdField, String dvField, String... ranges) { + List fieldsAndValues = new ArrayList<>(); + fieldsAndValues.add("id"); + fieldsAndValues.add(id); + for (String r : ranges) { + fieldsAndValues.add(bkdField); + fieldsAndValues.add(r); + fieldsAndValues.add(dvField); + fieldsAndValues.add(r); + } + assertU(adoc(fieldsAndValues.toArray(new String[0]))); + } + + /** Asserts the BKD field and the docValues field return the same (expected) count. */ + private void assertSameCount( + String criteria, String range, String bkdField, String dvField, int expected) { + for (String f : new String[] {bkdField, dvField}) { + assertQ( + "criteria=" + criteria + " field=" + f + " " + range, + req("q", "{!numericRange criteria=" + criteria + " field=" + f + "}" + range), + "//result[@numFound='" + expected + "']"); + } + } + + @Test + public void testContainsPerRangeNotUnion() { + // doc 1: two overlapping ranges whose *union* is [1,20] + addMvDoc("1", "price_range_multi", "price_range_mv_dv", "[1 TO 10]", "[5 TO 20]"); + // doc 2: two disjoint ranges + addMvDoc("2", "price_range_multi", "price_range_mv_dv", "[0 TO 5]", "[100 TO 200]"); + assertU(commit()); + + // Neither individual range of doc 1 contains [1,15] + assertSameCount("contains", "[1 TO 15]", "price_range_multi", "price_range_mv_dv", 0); + + // doc 2's [0,5] and [100,200] each satisfy one half-open bound of a CONTAINS([3,150]) split, so + // a decomposition would falsely match -- native per-range matching must not. + assertSameCount("contains", "[3 TO 150]", "price_range_multi", "price_range_mv_dv", 0); + + // doc 1's [1,10] actually contains [1,10] (any range suffices). + assertSameCount("contains", "[1 TO 10]", "price_range_multi", "price_range_mv_dv", 1); + } + + @Test + public void testAllRelationsMultiValued() { + addMvDoc("1", "price_range_multi", "price_range_mv_dv", "[1 TO 10]", "[5 TO 20]"); + addMvDoc("2", "price_range_multi", "price_range_mv_dv", "[0 TO 5]", "[100 TO 200]"); + addMvDoc("3", "price_range_multi", "price_range_mv_dv", "[130 TO 160]"); + assertU(commit()); + + // INTERSECTS [12,14]: only doc 1 (via its [5,20] range). + assertSameCount("intersects", "[12 TO 14]", "price_range_multi", "price_range_mv_dv", 1); + // WITHIN [0,25]: doc 1 ([1,10]) and doc 2 ([0,5]) each have a range within it. + assertSameCount("within", "[0 TO 25]", "price_range_multi", "price_range_mv_dv", 2); + // CROSSES [8,50]: doc 1 crosses (its ranges overlap but aren't within); docs 2,3 are disjoint. + assertSameCount("crosses", "[8 TO 50]", "price_range_multi", "price_range_mv_dv", 1); + } + + @Test + public void test2dLongMultiValued() { + addMvDoc( + "1", + "long_range_multi_2d", + "long_range_mv_dv_2d", + "[100,100 TO 200,200]", + "[500,500 TO 600,600]"); + addMvDoc("2", "long_range_multi_2d", "long_range_mv_dv_2d", "[1,3000000000 TO 5,4000000000]"); + assertU(commit()); + + assertSameCount( + "intersects", "[150,100 TO 550,150]", "long_range_multi_2d", "long_range_mv_dv_2d", 1); + assertSameCount( + "intersects", + "[2,3500000000 TO 5,3600000000]", + "long_range_multi_2d", + "long_range_mv_dv_2d", + 1); + } + + @Test + public void testFloatMultiValued() { + addMvDoc("1", "float_range_multi", "float_range_mv_dv", "[1.0 TO 2.0]", "[5.0 TO 6.0]"); + addMvDoc("2", "float_range_multi", "float_range_mv_dv", "[10.0 TO 20.0]"); + assertU(commit()); + + assertSameCount("intersects", "[1.5 TO 1.8]", "float_range_multi", "float_range_mv_dv", 1); + assertSameCount("within", "[0.0 TO 100.0]", "float_range_multi", "float_range_mv_dv", 2); + } + + @Test + public void test2dDoubleMultiValued() { + addMvDoc( + "1", + "double_range_multi_2d", + "double_range_mv_dv_2d", + "[1.0,-3.1 TO 5.0,-1.1]", + "[-5.5,10.0 TO 1.5,15.0]"); + addMvDoc("2", "double_range_multi_2d", "double_range_mv_dv_2d", "[-10,2.5 TO 0,3.5]"); + assertU(commit()); + + // Ranges per doc (each a 2D bounding box): + // doc1 A: D1 [1.0, 5.0] D2 [-3.1, -1.1] + // doc1 B: D1 [-5.5, 1.5] D2 [10.0, 15.0] + // doc2 C: D1 [-10.0, 0.0] D2 [2.5, 3.5] + + // CONTAINS: a doc range must wrap the query box in *both* dims. doc1's A wraps + // [2,4]x[-3.0,-1.15] (1<=2, 5>=4 and -3.1<=-3.0, -1.1>=-1.15). + // doc2's C does not (its D1 max 0 < 4). -> only doc 1. + assertSameCount( + "contains", "[2.0,-3.0 TO 4.0,-1.15]", "double_range_multi_2d", "double_range_mv_dv_2d", 1); + + // WITHIN: a doc range must fit *inside* the query box in both dims. doc1's A fits inside + // [0,6]x[-4,-1]; doc2's C sticks out on D1 (min -10 < 0). -> only doc 1. + assertSameCount( + "within", "[0.0,-4.0 TO 6.0,-1.0]", "double_range_multi_2d", "double_range_mv_dv_2d", 1); + + // INTERSECTS: overlap in both dims is enough. The central box [-2,3]x[-2,3] overlaps doc1's A + // and doc2's C, so both match (any-range / any-doc "OR" semantics). -> 2 docs. + assertSameCount( + "intersects", + "[-2.0,-2.0 TO 3.0,3.0]", + "double_range_multi_2d", + "double_range_mv_dv_2d", + 2); + + // CROSSES = intersects AND NOT within. For [-1,6]x[-4,3], doc1's A is fully *within* it (so + // does not cross) and its B is disjoint -> doc 1 excluded; doc2's C overlaps but pokes out on + // D1 (min -10) -> it crosses. -> only doc 2, which distinguishes crosses from within. + assertSameCount( + "crosses", "[-1.0,-4.0 TO 6.0,3.0]", "double_range_multi_2d", "double_range_mv_dv_2d", 1); + } +} diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc index 7a9508e083f8..d28820458aba 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc @@ -51,17 +51,17 @@ The {solr-javadocs}/core/org/apache/solr/schema/package-summary.html[`org.apache |IntPointField |Integer field (32-bit signed integer). This class encodes int values using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. For single valued fields, `docValues="true"` must be used to enable sorting. -|IntRangeField |Stores single or multi-dimensional ranges, using syntax like `[1 TO 4]` or `[1,2 TO 3,4]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type does not support docValues. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. +|IntRangeField |Stores single or multi-dimensional ranges, using syntax like `[1 TO 4]` or `[1,2 TO 3,4]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type supports `docValues="true"` (with `multiValued="true"` or `"false"`) as a query-time filter optimization. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. |LatLonPointSpatialField |A latitude/longitude coordinate pair; possibly multi-valued for multiple points. Usually it's specified as "lat,lon" order with a comma. See the section xref:query-guide:spatial-search.adoc[] for more information. |LongPointField |Long field (64-bit signed integer). This class encodes foo values using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. For single valued fields, `docValues="true"` must be used to enable sorting. -|LongRangeField |Stores single or multi-dimensional ranges of long integers, using syntax like `[1000000000 TO 4000000000]` or `[1,2 TO 3,4]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type does not support docValues. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. +|LongRangeField |Stores single or multi-dimensional ranges of long integers, using syntax like `[1000000000 TO 4000000000]` or `[1,2 TO 3,4]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type supports `docValues="true"` (with `multiValued="true"` or `"false"`) as a query-time filter optimization. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. -|DoubleRangeField |Stores single or multi-dimensional ranges of double-precision floating-point numbers, using syntax like `[1.5 TO 2.5]` or `[1.0,2.0 TO 3.0,4.0]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type does not support docValues. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. +|DoubleRangeField |Stores single or multi-dimensional ranges of double-precision floating-point numbers, using syntax like `[1.5 TO 2.5]` or `[1.0,2.0 TO 3.0,4.0]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type supports `docValues="true"` (with `multiValued="true"` or `"false"`) as a query-time filter optimization. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. -|FloatRangeField |Stores single or multi-dimensional ranges of floating-point numbers, using syntax like `[1.5 TO 4.5]` or `[1.0,2.0 TO 3.0,4.0]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type does not support docValues. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. +|FloatRangeField |Stores single or multi-dimensional ranges of floating-point numbers, using syntax like `[1.5 TO 4.5]` or `[1.0,2.0 TO 3.0,4.0]`. Up to 4 dimensions are supported. Dimensionality is specified on new field-types using a `numDimensions` property, and all values for a particular field must have exactly this number of dimensions. Field type supports `docValues="true"` (with `multiValued="true"` or `"false"`) as a query-time filter optimization. Typically queried using the xref:query-guide:other-parsers.adoc#numeric-range-query-parser[Numeric Range Query Parser], though the Lucene and other query parsers also support this field by assuming "contains" semantics for searches. |NestPathField | Specialized field type storing enhanced information, when xref:indexing-nested-documents.adoc#schema-configuration[working with nested documents].