diff --git a/changelog/unreleased/score-join-numeric-fields.yml b/changelog/unreleased/score-join-numeric-fields.yml
new file mode 100644
index 00000000000..b1b1c511c2d
--- /dev/null
+++ b/changelog/unreleased/score-join-numeric-fields.yml
@@ -0,0 +1,9 @@
+title: The `{!join score=...}` query parser now supports joining `from` `NUMERIC` or `SORTED_NUMERIC` doc values `to` numeric Point fields (`IntPointField`, `LongPointField`, `FloatPointField`, `DoublePointField`, `DatePointField`).
+type: added
+authors:
+ - name: Mikhail Khludnev
+ nick: mkhludnev
+ email: mkhl@apache.org
+links:
+ - name: GITHUB#4880
+ url: https://github.com/apache/solr/pull/4880
diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
index 41f3dc56328..d97b7681cd9 100644
--- a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java
@@ -47,6 +47,9 @@
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.NumberType;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.JoinQParserPlugin;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
@@ -60,9 +63,11 @@
/**
* Create a query-time join query with scoring. It just calls {@link
* JoinUtil#createJoinQuery(String, boolean, String, Query, org.apache.lucene.search.IndexSearcher,
- * ScoreMode)}. It runs subordinate query and collects values of "from" field and scores, then it
- * lookups these collected values in "to" field, and yields aggregated scores. Local parameters are
- * similar to {@link JoinQParserPlugin} {!join}
* This plugin doesn't have its own name, and is called by specifying local parameter
* {!join score=...}.... Note: this parser is invoked even if you specify score=none
@@ -75,10 +80,11 @@
* type="string" docValues="true". note: if docValues
* are not enabled for this field, it will work anyway, but it costs some memory for {@link
- * UninvertingReader}. Also, numeric doc values are not supported until LUCENE-5868. Thus, it only
- * supports {@link DocValuesType#SORTED}, {@link DocValuesType#SORTED_SET}, {@link
- * DocValuesType#BINARY}.
+ * UninvertingReader}. Non-numeric fields only support {@link DocValuesType#SORTED}, {@link
+ * DocValuesType#SORTED_SET}, {@link DocValuesType#BINARY}. Numeric fields having {@link
+ * DocValuesType#NUMERIC} or {@link DocValuesType#SORTED_NUMERIC} doc values — such a
+ * field does not need to be indexed, only {@code docValues="true"} is required — in
+ * which case the matching "to" field must be indexed as the same numeric Point field type.
* fromIndex - optional parameter, a core name where subordinate query should run (and
* from values are collected) rather than current core.
* Example:q={!join from=manu_id_s to=id score=total fromIndex=products}foo
@@ -132,8 +138,14 @@ public Weight createWeight(
final Query joinQuery;
try {
joinQuery =
- JoinUtil.createJoinQuery(
- fromField, true, toField, fromQuery, fromHolder.get(), this.scoreMode);
+ createJoinQuery(
+ fromField,
+ fromCore.getLatestSchema(),
+ toField,
+ info.getReq().getSchema(),
+ fromQuery,
+ fromHolder.get(),
+ this.scoreMode);
} finally {
fromCore.close();
fromHolder.decref();
@@ -189,9 +201,16 @@ public Weight createWeight(
IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost)
throws IOException {
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+ final IndexSchema schema = info.getReq().getSchema();
final Query jq =
- JoinUtil.createJoinQuery(
- fromField, true, toField, fromQuery, info.getReq().getSearcher(), this.scoreMode);
+ createJoinQuery(
+ fromField,
+ schema,
+ toField,
+ schema,
+ fromQuery,
+ info.getReq().getSearcher(),
+ this.scoreMode);
return jq.rewrite(searcher).createWeight(searcher, scoreMode, boost);
}
@@ -237,6 +256,94 @@ public void visit(QueryVisitor visitor) {
}
}
+ /**
+ * Creates a join query, delegating to {@link JoinUtil#createJoinQuery(String, boolean, String,
+ * Class, Query, IndexSearcher, ScoreMode)} for a numeric {@code fromField}/Point {@code toField}
+ * pair, or to {@link JoinUtil#createJoinQuery(String, boolean, String, Query, IndexSearcher,
+ * ScoreMode)} otherwise.
+ *
+ * @param fromField "foreign key" field name; any field type with a numeric {@link NumberType}
+ * (not necessarily a Point field type) qualifies, as long as {@code docValues="true"} is set.
+ * It doesn't need to be {@code indexed}.
+ * @param fromSchema schema holding {@code fromField}, used to detect numeric doc values
+ * @param toField "primary key" field name
+ * @param toSchema schema holding {@code toField}, used to detect numeric Point fields
+ * @param fromQuery the query to match documents on the from side
+ * @param fromSearcher the searcher that executed the specified fromQuery
+ * @param scoreMode instructs how scores from the fromQuery are mapped to the returned query
+ * @return a {@link Query} instance that can be used to join documents based on the values in the
+ * from and to field
+ */
+ static Query createJoinQuery(
+ String fromField,
+ IndexSchema fromSchema,
+ String toField,
+ IndexSchema toSchema,
+ Query fromQuery,
+ IndexSearcher fromSearcher,
+ ScoreMode scoreMode)
+ throws IOException {
+ final SchemaField fromSchemaField = fromSchema.getFieldOrNull(fromField);
+ final NumberType fromNumberType =
+ fromSchemaField == null ? null : fromSchemaField.getType().getNumberType();
+ if (fromNumberType != null) {
+ if (!fromSchemaField.hasDocValues()) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Numeric join 'from' field '"
+ + fromField
+ + "' must have docValues enabled; it doesn't need to be indexed.");
+ }
+ final SchemaField toSchemaField = toSchema.getFieldOrNull(toField);
+ final boolean toIsPoint = toSchemaField != null && toSchemaField.getType().isPointField();
+ final NumberType toNumberType =
+ toSchemaField == null ? null : toSchemaField.getType().getNumberType();
+ if (!toIsPoint || fromNumberType != toNumberType) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Numeric join 'from' field '"
+ + fromField
+ + "' ("
+ + fromNumberType
+ + ") requires a 'to' field of the same numeric Point field type, but '"
+ + toField
+ + "' is "
+ + (toSchemaField == null ? "undefined" : toSchemaField.getType().getTypeName())
+ + ".");
+ }
+ return JoinUtil.createJoinQuery(
+ fromField,
+ fromSchemaField.multiValued(),
+ toField,
+ numericClass(fromNumberType),
+ fromQuery,
+ fromSearcher,
+ scoreMode);
+ }
+ return JoinUtil.createJoinQuery(fromField, true, toField, fromQuery, fromSearcher, scoreMode);
+ }
+
+ /** Maps a schema {@link NumberType} to the {@link Class} expected by {@link JoinUtil}. */
+ private static Class numericClass(NumberType numberType) {
+ switch (numberType) {
+ case INTEGER:
+ return Integer.class;
+ case LONG:
+ // DatePointField values are internally represented as milliseconds since epoch (long),
+ // via LongPoint, so DATE shares the same encoding as LONG.
+ case DATE:
+ return Long.class;
+ case FLOAT:
+ return Float.class;
+ case DOUBLE:
+ return Double.class;
+ default:
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "Unsupported numeric join field type " + numberType);
+ }
+ }
+
@Override
public QParser createParser(
String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
index 3cd370786ac..fb73c98e39e 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema12.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
@@ -42,6 +42,10 @@
+
+
+
@@ -750,6 +754,7 @@
+
@@ -761,6 +766,7 @@
+
diff --git a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java
index 662cda05439..3b312ab8105 100644
--- a/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java
+++ b/solr/core/src/test/org/apache/solr/search/join/TestScoreJoinQPScore.java
@@ -116,6 +116,192 @@ public void testSimple() throws Exception {
dir.close();*/
}
+ public void testNumericJoinSingleValued() throws Exception {
+ clearIndex();
+
+ // products
+ assertU(add(doc("name", "name1", idField, "1", "cat_pi", "100")));
+ assertU(add(doc("name", "name2", idField, "4", "cat_pi", "200")));
+
+ // offers, referencing the product via a numeric Point field
+ assertU(add(doc("price_s", "10.0", idField, "2", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "20.0", idField, "3", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "10.0", idField, "5", "prodRef_pi", "200")));
+ assertU(add(doc("price_s", "20.0", idField, "6", "prodRef_pi", "200")));
+
+ assertU(commit());
+
+ assertJQ(
+ req("q", "{!join from=cat_pi to=prodRef_pi score=None}name:name2", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'5'},{'id':'6'}]}");
+
+ assertJQ(
+ req("q", "{!join from=cat_pi to=prodRef_pi score=None}name:name1", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'2'},{'id':'3'}]}");
+
+ // reverse direction: from Point ("to" side of a Point field can also serve as the numeric
+ // docValues "from" side for the join, since pint fields have both indexed points & docValues)
+ assertJQ(
+ req("q", "{!join from=prodRef_pi to=cat_pi score=None}id:5", "fl", "id"),
+ "/response=={'numFound':1,'start':0,'numFoundExact':true,'docs':[{'id':'4'}]}");
+ }
+
+ public void testNumericJoinMultiValued() throws Exception {
+ clearIndex();
+
+ // products, each may belong to several categories (multi-valued numeric field)
+ assertU(add(doc("name", "name1", idField, "1", "cat_pis", "100", "cat_pis", "300")));
+ assertU(add(doc("name", "name2", idField, "4", "cat_pis", "200")));
+
+ // offers, referencing a single category
+ assertU(add(doc("price_s", "10.0", idField, "2", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "20.0", idField, "3", "prodRef_pi", "300")));
+ assertU(add(doc("price_s", "10.0", idField, "5", "prodRef_pi", "200")));
+
+ assertU(commit());
+
+ assertJQ(
+ req("q", "{!join from=cat_pis to=prodRef_pi score=None}name:name1", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'2'},{'id':'3'}]}");
+
+ assertJQ(
+ req("q", "{!join from=cat_pis to=prodRef_pi score=None}name:name2", "fl", "id"),
+ "/response=={'numFound':1,'start':0,'numFoundExact':true,'docs':[{'id':'5'}]}");
+ }
+
+ public void testNumericJoinWithScoring() throws Exception {
+ clearIndex();
+
+ assertU(
+ add(
+ doc(
+ "t_description",
+ "A random movie",
+ "name",
+ "Movie 1",
+ idField,
+ "1",
+ "movieId_pi",
+ "10")));
+ assertU(
+ add(doc("title", "The first subtitle of this movie", idField, "2", "prodRef_pi", "10")));
+ assertU(
+ add(doc("title", "random subtitle; random event movie", idField, "3", "prodRef_pi", "10")));
+ assertU(
+ add(
+ doc(
+ "t_description",
+ "A second random movie",
+ "name",
+ "Movie 2",
+ idField,
+ "4",
+ "movieId_pi",
+ "20")));
+ assertU(
+ add(
+ doc(
+ "title",
+ "a very random event happened during christmas night",
+ idField,
+ "5",
+ "prodRef_pi",
+ "20")));
+ assertU(commit());
+
+ assertJQ(
+ req("q", "{!join from=prodRef_pi to=movieId_pi score=Max}title:random", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'1'},{'id':'4'}]}");
+ }
+
+ public void testNumericJoinDateField() throws Exception {
+ clearIndex();
+
+ // products, referenced by a release date (pdate uses the same Long encoding as plong)
+ assertU(add(doc("name", "name1", idField, "1", "releaseDate_pdt", "2020-01-01T00:00:00Z")));
+ assertU(add(doc("name", "name2", idField, "4", "releaseDate_pdt", "2021-06-15T00:00:00Z")));
+
+ // offers, referencing the product via the same date value
+ assertU(add(doc("price_s", "10.0", idField, "2", "prodDate_pdt", "2020-01-01T00:00:00Z")));
+ assertU(add(doc("price_s", "20.0", idField, "3", "prodDate_pdt", "2020-01-01T00:00:00Z")));
+ assertU(add(doc("price_s", "10.0", idField, "5", "prodDate_pdt", "2021-06-15T00:00:00Z")));
+
+ assertU(commit());
+
+ assertJQ(
+ req("q", "{!join from=releaseDate_pdt to=prodDate_pdt score=None}name:name1", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'2'},{'id':'3'}]}");
+
+ assertJQ(
+ req("q", "{!join from=releaseDate_pdt to=prodDate_pdt score=None}name:name2", "fl", "id"),
+ "/response=={'numFound':1,'start':0,'numFoundExact':true,'docs':[{'id':'5'}]}");
+ }
+
+ public void testNumericJoinFromNonIndexedDocValues() throws Exception {
+ clearIndex();
+
+ // products: "cat_ii" is declared indexed="false", so it only carries numeric doc values,
+ // no indexed points at all; the numeric join must still work off doc values alone.
+ assertU(add(doc("name", "name1", idField, "1", "cat_ii", "100")));
+ assertU(add(doc("name", "name2", idField, "4", "cat_ii", "200")));
+
+ // offers, referencing the product via an indexed numeric Point field
+ assertU(add(doc("price_s", "10.0", idField, "2", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "20.0", idField, "3", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "10.0", idField, "5", "prodRef_pi", "200")));
+ assertU(add(doc("price_s", "20.0", idField, "6", "prodRef_pi", "200")));
+
+ assertU(commit());
+
+ assertJQ(
+ req("q", "{!join from=cat_ii to=prodRef_pi score=None}name:name2", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'5'},{'id':'6'}]}");
+
+ assertJQ(
+ req("q", "{!join from=cat_ii to=prodRef_pi score=None}name:name1", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'2'},{'id':'3'}]}");
+ }
+
+ public void testNumericJoinFromLegacyTrieField() throws Exception {
+ clearIndex();
+
+ // products: "cat_trie_i" is a legacy (non-Point) TrieIntField with docValues, not indexed;
+ // the numeric join's "from" side only relies on numeric doc values, not on Point encoding.
+ assertU(add(doc("name", "name1", idField, "1", "cat_trie_i", "100")));
+ assertU(add(doc("name", "name2", idField, "4", "cat_trie_i", "200")));
+
+ // offers, referencing the product via an indexed numeric Point field
+ assertU(add(doc("price_s", "10.0", idField, "2", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "20.0", idField, "3", "prodRef_pi", "100")));
+ assertU(add(doc("price_s", "10.0", idField, "5", "prodRef_pi", "200")));
+ assertU(add(doc("price_s", "20.0", idField, "6", "prodRef_pi", "200")));
+
+ assertU(commit());
+
+ assertJQ(
+ req("q", "{!join from=cat_trie_i to=prodRef_pi score=None}name:name2", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'5'},{'id':'6'}]}");
+
+ assertJQ(
+ req("q", "{!join from=cat_trie_i to=prodRef_pi score=None}name:name1", "fl", "id"),
+ "/response=={'numFound':2,'start':0,'numFoundExact':true,'docs':[{'id':'2'},{'id':'3'}]}");
+ }
+
+ public void testNumericJoinTypeMismatch() throws Exception {
+ clearIndex();
+ assertU(add(doc("name", "name1", idField, "1", "cat_pi", "100")));
+ assertU(add(doc("price_s", "10.0", idField, "2", "prodRef_pl", "100")));
+ assertU(commit());
+
+ // "from" is an int field, "to" is a long point field: types don't match, a clear error is
+ // raised instead of a low-level Lucene point encoding failure.
+ assertQEx(
+ "numeric join type mismatch",
+ "Numeric join",
+ req("q", "{!join from=cat_pi to=prodRef_pl score=None}name:name1", "fl", "id"),
+ SolrException.ErrorCode.BAD_REQUEST);
+ }
+
public void testDeleteByScoreJoinQuery() throws Exception {
indexDataForScoring();
String joinQuery = "{!join from=" + toField + " to=" + idField + " score=Max}title:random";
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/join-query-parser.adoc b/solr/solr-ref-guide/modules/query-guide/pages/join-query-parser.adoc
index ac39ceed751..bf1cacd8324 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/join-query-parser.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/join-query-parser.adoc
@@ -121,11 +121,12 @@ The first access to the field cache slows down the initial requests following a
Performance scales linearly with the number of values matched in the "from" field.
This method must be used if score information is required, and should also be considered when the "from" query matches few documents, regardless of the number of "to" side documents returned.
+
-.dvWithScore and single value numerics
-[WARNING]
+.dvWithScore and numerics
+[NOTE]
====
-The `dvWithScore` method doesn't support single value numeric fields.
-Users migrating from versions prior to 7.0 are encouraged to change field types to string and rebuild indexes during migration.
+The `dvWithScore` method supports numeric fields on the "from" side that have `docValues="true"` (single- or multi-valued), regardless of whether the field type is a `Point` field (`IntPointField`, `LongPointField`, `FloatPointField`, `DoublePointField`, `DatePointField`) or a legacy `Trie` numeric field, as long as the "to" side field uses the matching numeric `Point` field type.
+The "from" field only needs `docValues="true"`; it does not need to be `indexed="true"`.
+The "to" field, however, must be indexed as a `Point` field, since it is queried by point value.
====
`topLevelDV`::: Can only be used when `to` and `from` fields have docValues data, and does not currently support numeric fields.