enable Lucene join to support numerics_dv -> points

changelog/unreleased/score-join-numeric-fields.yml

@@ -0,0 +1,9 @@
+title: The `{!join score=...}` query parser now supports joining `from` `NUMERIC` or `SORTED_NUMERIC` doc values `to` numeric Point fields (`IntPointField`, `LongPointField`, `FloatPointField`, `DoublePointField`, `DatePointField`).
+type: added
+authors:
+  - name: Mikhail Khludnev
+    nick: mkhludnev
+    email: mkhl@apache.org
+links:
+  - name: GITHUB#4880
+    url: https://github.com/apache/solr/pull/4880

solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java

@@ -47,6 +47,9 @@
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.NumberType;
+import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.JoinQParserPlugin;
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.QParserPlugin;
@@ -60,9 +63,11 @@
 /**
  * Create a query-time join query with scoring. It just calls {@link
  * JoinUtil#createJoinQuery(String, boolean, String, Query, org.apache.lucene.search.IndexSearcher,
- * ScoreMode)}. It runs subordinate query and collects values of "from" field and scores, then it
- * lookups these collected values in "to" field, and yields aggregated scores. Local parameters are
- * similar to {@link JoinQParserPlugin} <a
+ * ScoreMode)} or, when the "from" field holds numeric doc values, {@link
+ * JoinUtil#createJoinQuery(String, boolean, String, Class, Query,
+ * org.apache.lucene.search.IndexSearcher, ScoreMode)}. It runs subordinate query and collects
+ * values of "from" field and scores, then it lookups these collected values in "to" field, and
+ * yields aggregated scores. Local parameters are similar to {@link JoinQParserPlugin} <a
  * href="https://solr.apache.org/guide/solr/latest/query-guide/join-query-parser.html">{!join}</a>
  * This plugin doesn't have its own name, and is called by specifying local parameter <code>
  * {!join score=...}...</code>. Note: this parser is invoked even if you specify <code>score=none
@@ -75,10 +80,11 @@
  *       <code>type="string" docValues="true"</code>. note: if <a
  *       href="https://solr.apache.org/guide/solr/latest/indexing-guide/docvalues.html">docValues</a>
  *       are not enabled for this field, it will work anyway, but it costs some memory for {@link
- *       UninvertingReader}. Also, numeric doc values are not supported until <a
- *       href="https://issues.apache.org/jira/browse/LUCENE-5868">LUCENE-5868</a>. Thus, it only
- *       supports {@link DocValuesType#SORTED}, {@link DocValuesType#SORTED_SET}, {@link
- *       DocValuesType#BINARY}.
+ *       UninvertingReader}. Non-numeric fields only support {@link DocValuesType#SORTED}, {@link
+ *       DocValuesType#SORTED_SET}, {@link DocValuesType#BINARY}. Numeric fields having {@link
+ *       DocValuesType#NUMERIC} or {@link DocValuesType#SORTED_NUMERIC} doc values &#8212; such a
+ *       field does not need to be indexed, only {@code docValues="true"} is required &#8212; in
+ *       which case the matching "to" field must be indexed as the same numeric Point field type.
  *   <li>fromIndex - optional parameter, a core name where subordinate query should run (and <code>
  *       from</code> values are collected) rather than current core. <br>
  *       Example:<code>q={!join from=manu_id_s to=id score=total fromIndex=products}foo</code>
@@ -132,8 +138,14 @@ public Weight createWeight(
       final Query joinQuery;
       try {
         joinQuery =
-            JoinUtil.createJoinQuery(
-                fromField, true, toField, fromQuery, fromHolder.get(), this.scoreMode);
+            createJoinQuery(
+                fromField,
+                fromCore.getLatestSchema(),
+                toField,
+                info.getReq().getSchema(),
+                fromQuery,
+                fromHolder.get(),
+                this.scoreMode);
       } finally {
         fromCore.close();
         fromHolder.decref();
@@ -189,9 +201,16 @@ public Weight createWeight(
         IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost)
         throws IOException {
       SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+      final IndexSchema schema = info.getReq().getSchema();
       final Query jq =
-          JoinUtil.createJoinQuery(
-              fromField, true, toField, fromQuery, info.getReq().getSearcher(), this.scoreMode);
+          createJoinQuery(
+              fromField,
+              schema,
+              toField,
+              schema,
+              fromQuery,
+              info.getReq().getSearcher(),
+              this.scoreMode);
       return jq.rewrite(searcher).createWeight(searcher, scoreMode, boost);
     }
 
@@ -237,6 +256,94 @@ public void visit(QueryVisitor visitor) {
     }
   }
 
+  /**
+   * Creates a join query, delegating to {@link JoinUtil#createJoinQuery(String, boolean, String,
+   * Class, Query, IndexSearcher, ScoreMode)} for a numeric {@code fromField}/Point {@code toField}
+   * pair, or to {@link JoinUtil#createJoinQuery(String, boolean, String, Query, IndexSearcher,
+   * ScoreMode)} otherwise.
+   *
+   * @param fromField "foreign key" field name; any field type with a numeric {@link NumberType}
+   *     (not necessarily a Point field type) qualifies, as long as {@code docValues="true"} is set.
+   *     It doesn't need to be {@code indexed}.
+   * @param fromSchema schema holding {@code fromField}, used to detect numeric doc values
+   * @param toField "primary key" field name
+   * @param toSchema schema holding {@code toField}, used to detect numeric Point fields
+   * @param fromQuery the query to match documents on the from side
+   * @param fromSearcher the searcher that executed the specified fromQuery
+   * @param scoreMode instructs how scores from the fromQuery are mapped to the returned query
+   * @return a {@link Query} instance that can be used to join documents based on the values in the
+   *     from and to field
+   */
+  static Query createJoinQuery(
+      String fromField,
+      IndexSchema fromSchema,
+      String toField,
+      IndexSchema toSchema,
+      Query fromQuery,
+      IndexSearcher fromSearcher,
+      ScoreMode scoreMode)
+      throws IOException {
+    final SchemaField fromSchemaField = fromSchema.getFieldOrNull(fromField);
+    final NumberType fromNumberType =
+        fromSchemaField == null ? null : fromSchemaField.getType().getNumberType();
+    if (fromNumberType != null) {
+      if (!fromSchemaField.hasDocValues()) {
+        throw new SolrException(
+            SolrException.ErrorCode.BAD_REQUEST,
+            "Numeric join 'from' field '"
+                + fromField
+                + "' must have docValues enabled; it doesn't need to be indexed.");
+      }
+      final SchemaField toSchemaField = toSchema.getFieldOrNull(toField);
+      final boolean toIsPoint = toSchemaField != null && toSchemaField.getType().isPointField();
+      final NumberType toNumberType =
+          toSchemaField == null ? null : toSchemaField.getType().getNumberType();
+      if (!toIsPoint || fromNumberType != toNumberType) {
+        throw new SolrException(
+            SolrException.ErrorCode.BAD_REQUEST,
+            "Numeric join 'from' field '"
+                + fromField
+                + "' ("
+                + fromNumberType
+                + ") requires a 'to' field of the same numeric Point field type, but '"
+                + toField
+                + "' is "
+                + (toSchemaField == null ? "undefined" : toSchemaField.getType().getTypeName())
+                + ".");
+      }
+      return JoinUtil.createJoinQuery(
+          fromField,
+          fromSchemaField.multiValued(),
+          toField,
+          numericClass(fromNumberType),
+          fromQuery,
+          fromSearcher,
+          scoreMode);
+    }
+    return JoinUtil.createJoinQuery(fromField, true, toField, fromQuery, fromSearcher, scoreMode);
+  }
+
+  /** Maps a schema {@link NumberType} to the {@link Class} expected by {@link JoinUtil}. */
+  private static Class<? extends Number> numericClass(NumberType numberType) {
+    switch (numberType) {
+      case INTEGER:
+        return Integer.class;
+      case LONG:
+      // DatePointField values are internally represented as milliseconds since epoch (long),
+      // via LongPoint, so DATE shares the same encoding as LONG.
+      case DATE:
+        return Long.class;
+      case FLOAT:
+        return Float.class;
+      case DOUBLE:
+        return Double.class;
+      default:
+        throw new SolrException(
+            SolrException.ErrorCode.BAD_REQUEST,
+            "Unsupported numeric join field type " + numberType);
+    }
+  }
+
   @Override
   public QParser createParser(
       String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {

solr/core/src/test-files/solr/collection1/conf/schema12.xml

@@ -42,6 +42,10 @@
   <fieldType name="tfloat" class="${solr.tests.FloatFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
   <fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
   <fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+
+  <!-- explicit (non-templated) legacy Trie numeric field type with docValues, used to test that
+       non-Point numeric fields can serve as the "from" side of a numeric score join -->
+  <fieldType name="trieint_dv" class="solr.TrieIntField" docValues="true" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
 
   <!-- Point Fields -->
   <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
@@ -750,6 +754,7 @@
   <dynamicField name="*_pf" type="pfloat" indexed="true" multiValued="false"/>
   <dynamicField name="*_pd" type="pdouble" indexed="true" multiValued="false"/>
   <dynamicField name="*_pdt" type="pdate" indexed="true" multiValued="false"/>
+  <dynamicField name="*_pis" type="pint" indexed="true" multiValued="true"/>
 
   <!-- some trie-coded dynamic fields for faster range queries -->
   <dynamicField name="*_ti" type="tint" indexed="true" stored="true"/>
@@ -761,6 +766,7 @@
   <dynamicField name="*_ii" type="pint" indexed="false" stored="false" useDocValuesAsStored="true"/>
   <dynamicField name="*_iis" type="pint" indexed="false" stored="false" useDocValuesAsStored="true"/>
   <dynamicField name="*_ff" type="pfloat" indexed="false" stored="false" useDocValuesAsStored="false"/>
+  <dynamicField name="*_trie_i" type="trieint_dv" indexed="false" stored="false" useDocValuesAsStored="true"/>
 
   <!-- testing fields with & without norms
        TODO: Remove numeric norms for SOLR-14199 -->