Updates cross-language-testing

pom.xml

@@ -135,10 +135,16 @@ under the License.
     <testng.version>7.12.0</testng.version>
     <org-slf4j.version>2.0.12</org-slf4j.version> <!-- used by TestNG -->
     <!-- these are TestNG groups used for excluding / including groups of tests. See profiles section. -->
-    <testng.generate-java-files>generate_java_files</testng.generate-java-files>
-    <testng.check-cpp-files>check_cpp_files</testng.check-cpp-files>
-    <testng.check-go-files>check_go_files</testng.check-go-files>
-    <testng.check-cpp-historical-files>check_cpp_historical_files</testng.check-cpp-historical-files>
+    <testng.generate_java_files>generate_java_files</testng.generate_java_files>
+    <testng.check_java_files>check_java_files</testng.check_java_files>
+    <testng.check_cpp_files>check_cpp_files</testng.check_cpp_files>
+    <testng.check_go_files>check_go_files</testng.check_go_files>
+    <testng.check_rust_files>check_rust_files</testng.check_rust_files>
+    <testng.check_cpp_historical_files>check_cpp_historical_files</testng.check_cpp_historical_files>
+    <testng.all_groups>${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files},${testng.check_go_files},${testng.check_rust_files},${testng.check_cpp_historical_files}</testng.all_groups>
+
+    <test.data.root>${project.basedir}</test.data.root>
+    <project.root>${project.basedir}</project.root>
 
     <!-- JVM arguments -->
     <jvm.mem>-Xmx4g</jvm.mem>
@@ -373,10 +379,21 @@ under the License.
           <version>${maven-javadoc-plugin.version}</version>
           <configuration>
             <locale>en_US</locale>
+            <overview>${project.basedir}/src/main/javadoc/overview.html</overview>
             <outputDirectory>${project.reporting.outputDirectory}</outputDirectory>
             <docfilessubdirs>true</docfilessubdirs>
             <show>public</show>
-            <doclint>none</doclint> <!-- use Checkstyle -->
+            <!-- Checkstyle's own Java grammar lags real JDK releases (e.g. it cannot parse
+                 JEP 513 flexible constructor bodies, standard since Java 25), so Javadoc
+                 correctness is enforced here instead, via doclint, which ships inside javac/javadoc
+                 itself and is always in sync with whatever JDK this project targets.
+                 Combined with <show>public</show> above, this execution (attach-javadocs, below)
+                 both publishes and fully validates (including completeness) only the public API
+                 surface that actually appears in the published Javadoc jar.
+                 A second execution (validate-internal-javadoc, below) separately checks every
+                 protected/package/private member too, for correctness (only doclint's "missing"
+                 category is dropped there), without publishing or requiring documentation on them. -->
+            <doclint>all</doclint>
             <quiet>true</quiet>
             <additionalJOptions>
               <additionalJOption>-J${jvm.mem}</additionalJOption>
@@ -412,6 +429,22 @@ under the License.
               <phase>package</phase>
               <goals><goal>jar</goal></goals>
             </execution>
+            <execution>
+              <!-- Validates Javadoc correctness (broken HTML, bad @param references, etc.) on
+                   protected/package/private members too, which <show>public</show> above never
+                   sees. Deliberately drops doclint's "missing" category so it never demands
+                   documentation exist on non-public members; it only requires that whatever
+                   documentation IS there is well-formed. Output is thrown away; nothing here
+                   is published. -->
+              <id>validate-internal-javadoc</id>
+              <phase>verify</phase>
+              <goals><goal>javadoc</goal></goals>
+              <configuration>
+                <show>private</show>
+                <doclint>all,-missing</doclint>
+                <outputDirectory>${project.build.directory}/javadoc-internal-check</outputDirectory>
+              </configuration>
+            </execution>
           </executions>
         </plugin>
 
@@ -471,15 +504,22 @@ under the License.
             <trimStackTrace>false</trimStackTrace>
             <useFile>true</useFile>
             <useManifestOnlyJar>false</useManifestOnlyJar>
-            <redirectTestOutputToFile>true</redirectTestOutputToFile>
+            <redirectTestOutputToFile>false</redirectTestOutputToFile>
             <reportsDirectory>${project.build.directory}/test-output/${maven.build.timestamp}</reportsDirectory>
             <perCoreThreadCount>false</perCoreThreadCount>
+            <suiteXmlFiles>
+              <suiteXmlFile>${project.basedir}/src/test/resources/testng.xml</suiteXmlFile>
+            </suiteXmlFiles>
+            <excludedGroups>${testng.all_groups}</excludedGroups>
             <properties>
               <property>
-                <name>verbose</name>
-                <value>0</value> 
+                <name>verbose</name><value>0</value>
               </property>
             </properties>
+            <systemPropertyVariables>
+              <test.data.root>${test.data.root}</test.data.root>
+              <project.root>${project.root}</project.root>
+            </systemPropertyVariables>
           </configuration>
         </plugin>
 
@@ -720,63 +760,111 @@ under the License.
     </profile>
 
     <profile>
-      <id>generate-java-files</id>
+      <id>generate_java_files</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <groups>${testng.generate_java_files}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+    <profile>
+      <id>check_all_language_files</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <groups>${testng.all_groups}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+    <profile>
+      <id>check_java_files</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <groups>${testng.generate_java_files},${testng.check_java_files}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
+    <profile>
+      <id>check_cpp_files</id>
       <build>
         <plugins>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <groups>${testng.generate-java-files}</groups>
-              <excludedGroups>${testng.check-cpp-files},${testng.check-go-files},${testng.check-cpp-historical-files}</excludedGroups>
+              <groups>${testng.check_cpp_files}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
             </configuration>
           </plugin>
         </plugins>
       </build>
     </profile>
 
     <profile>
-      <id>check-cpp-files</id>
+      <id>check_go_files</id>
       <build>
         <plugins>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <groups>${testng.check-cpp-files}</groups>
-              <excludedGroups>${testng.generate-java-files},${testng.check-go-files},${testng.check-cpp-historical-files}</excludedGroups>
+              <groups>${testng.check_go_files}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
             </configuration>
           </plugin>
         </plugins>
       </build>
     </profile>
 
     <profile>
-      <id>check-go-files</id>
+      <id>check_rust_files</id>
       <build>
         <plugins>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <groups>${testng.check-go-files}</groups>
-              <excludedGroups>${testng.generate-java-files},${testng.check-cpp-files},${testng.check-cpp-historical-files}</excludedGroups>
+              <groups>${testng.check_rust_files}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
             </configuration>
           </plugin>
         </plugins>
       </build>
     </profile>
 
     <profile>
-      <id>check-cpp-historical-files</id>
+      <id>check_cpp_historical_files</id>
       <build>
         <plugins>
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
-              <groups>${testng.check-cpp-historical-files}</groups>
-              <excludedGroups>${testng.generate-java-files},${testng.check-go-files},${testng.check-cpp-files}</excludedGroups>
+              <groups>${testng.check_cpp_historical_files}</groups>
+              <excludedGroups combine.self="override"></excludedGroups>
             </configuration>
           </plugin>
         </plugins>

src/main/java/org/apache/datasketches/common/ArrayOfStringsSerDe.java

@@ -35,8 +35,6 @@
  * this method is 2 times more compact, but it takes more time to encode and decode
  * by a factor of 1.5 to 2.
  *
- * <p>The serialization
- *
  * @author Alexander Saydakov
  */
 public class ArrayOfStringsSerDe extends ArrayOfItemsSerDe<String> {

src/main/java/org/apache/datasketches/common/BoundsOnRatiosInSampledSets.java

@@ -47,7 +47,7 @@ private BoundsOnRatiosInSampledSets() {}
    * @param f the inclusion probability used to produce the set with size <i>a</i> and should
    * generally be less than 0.5. Above this value, the results not be reliable.
    * When <i>f</i> = 1.0 this returns the estimate.
-   * @return the approximate upper bound
+   * @return the approximate lower bound
    */
   public static double getLowerBoundForBoverA(final long a, final long b, final double f) {
     checkInputs(a, b, f);
@@ -61,7 +61,7 @@ public static double getLowerBoundForBoverA(final long a, final long b, final do
    * @param a See class javadoc
    * @param b See class javadoc
    * @param f the inclusion probability used to produce the set with size <i>a</i>.
-   * @return the approximate lower bound
+   * @return the approximate upper bound
    */
   public static double getUpperBoundForBoverA(final long a, final long b, final double f) {
     checkInputs(a, b, f);
@@ -86,7 +86,7 @@ public static double getEstimateOfBoverA(final long a, final long b) {
    * Return the estimate of A. See class javadoc.
    * @param a See class javadoc
    * @param f the inclusion probability used to produce the set with size <i>a</i>.
-   * @return the approximate lower bound
+   * @return the approximate estimate of A.
    */
   public static double getEstimateOfA(final long a, final double f) {
     checkInputs(a, 1, f);
@@ -97,7 +97,7 @@ public static double getEstimateOfA(final long a, final double f) {
    * Return the estimate of B. See class javadoc.
    * @param b See class javadoc
    * @param f the inclusion probability used to produce the set with size <i>b</i>.
-   * @return the approximate lower bound
+   * @return the approximate estimate of B.
    */
   public static double getEstimateOfB(final long b, final double f) {
     checkInputs(b + 1, b, f);

src/main/java/org/apache/datasketches/common/MemorySegmentRequestExample.java

@@ -40,7 +40,7 @@ public final class MemorySegmentRequestExample implements MemorySegmentRequest {
   public MemorySegmentRequestExample() { }
 
   /**
-   * Request a new off-heap MemorySegment with the given <i>newByteSeze</i>.
+   * Request a new off-heap MemorySegment with the given <i>newByteSize</i>.
    * An internal confined Arena is created to exclusively manage the new segment and it is associated
    * with the new segment with a ConcurrentHashMap.
    */

src/main/java/org/apache/datasketches/hll/AbstractCoupons.java

@@ -51,8 +51,8 @@ abstract class AbstractCoupons extends HllSketchImpl {
    * <p>Note: This is an approximation to the true mapping from numCoupons to N,
    * which has a range of validity roughly from 0 to 6 million coupons.</p>
    *
-   * <p>The k of the implied coupon sketch, which must not be confused with the k of the HLL
-   * sketch.  In this application k is always 2^26, which is the number of address bits of the
+   * <p>The k of the implied coupon sketch must not be confused with the k of the HLL
+   * sketch.  In this context k is always 2^26, which is the number of address bits of the
    * 32-bit coupon.</p>
    * @return the unique count estimate.
    */

src/main/java/org/apache/datasketches/hll/HllSketch.java

@@ -81,8 +81,8 @@
  *
  * <pre>{@code
  * double estimate = sketch.getEstimate();
- * double estUB = sketch.getUpperBound(1.0); //the upper bound at 1 standard deviation.
- * double estLB = sketch.getLowerBound(1.0); //the lower bound at 1 standard deviation.
+ * double estUB = sketch.getUpperBound(1); //the upper bound at 1 standard deviation.
+ * double estLB = sketch.getLowerBound(1); //the lower bound at 1 standard deviation.
  * //OR
  * System.out.println(sketch.toString()); //will output a summary of the sketch.
  * }</pre>

src/main/java/org/apache/datasketches/hll/PairIterator.java

@@ -36,13 +36,14 @@ String getHeader() {
    * In LIST and SET modes, this gets the iterating index into the integer array of HLL key/value
    * pairs.
    * In HLL mode, this is the iterating index into the hypothetical array of HLL values, which may
-   * be physically contructed differently based on the compaction scheme (HLL_4, HLL_6, HLL_8).
+   * be physically constructed differently based on the compaction scheme (HLL_4, HLL_6, HLL_8).
    * @return the index.
    */
   abstract int getIndex();
 
   /**
-   * Gets the key, the low 26 bits of an pair, and can be up to 26 bits in length.
+   * Gets the key from the low 26 bits of a pair.
+   * The key corresponds to the address in the HLL array and can be up to 26 bits in length.
    * @return the key
    */
   abstract int getKey();

src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java

@@ -86,7 +86,7 @@ class KllDirectDoublesSketch extends KllDoublesSketch {
    * @param k parameter that controls size of the sketch and accuracy of estimates
    * @param m parameter that controls the minimum level width in items.
    * @param dstSeg the given destination MemorySegment object for use by the sketch
-   * @param mSegReq the callback for the sketch to request a larger MemorySegment.
+   * @param mSegmentRequest the callback for the sketch to request a larger MemorySegment.
    * @return a new instance of this sketch
    */
   static KllDirectDoublesSketch newDirectUpdatableInstance(

src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java

@@ -86,7 +86,7 @@ class KllDirectLongsSketch extends KllLongsSketch {
    * @param k parameter that controls size of the sketch and accuracy of estimates
    * @param m parameter that controls the minimum level width in items.
    * @param dstSeg the given destination MemorySegment object for use by the sketch
-   * @param mSegReq the callback for the sketch to request a larger MemorySegment.
+   * @param mSegmentRequest the callback for the sketch to request a larger MemorySegment.
    * @return a new instance of this sketch
    */
   static KllDirectLongsSketch newDirectUpdatableInstance(

src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java

@@ -52,7 +52,7 @@
  * This allows the possibility of the introduction of different serialization
  * schemes with minimal impact on the rest of the library.</p>
  *
- * <h3>Visual Layout</h3>
+ * <h2>Visual Layout</h2>
  * The low significance bytes of the visual data structure below are on the left.
  * The multi-byte primitives are stored in native byte order.
  * The numeric <i>byte</i> and <i>short</i> fields are treated as unsigned.

src/main/java/org/apache/datasketches/quantiles/CompactQuantilesDoublesSketch.java

@@ -37,7 +37,7 @@ boolean isCompact() {
 
   @Override
   public boolean isReadOnly() {
-    return false;
+    return true;
   }
 
   @Override

src/main/java/org/apache/datasketches/quantiles/DoublesUtil.java

@@ -34,9 +34,7 @@
 /**
  * Utilities that support the doubles quantiles algorithms.
  *
- * <p>This class contains a highly specialized sort called blockyTandemMergeSort().
- * It also contains methods that are used while building histograms and other common
- * functions.</p>
+ * <p>This class contains methods that are used while building histograms and other common functions.</p>
  *
  * @author Lee Rhodes
  */

src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java

@@ -219,8 +219,7 @@ private static <T> void justZipWithStride(
    * blockyTandemMergeSort() is an implementation of top-down merge sort specialized
    * for the case where the input contains successive equal-length blocks
    * that have already been sorted, so that only the top part of the
-   * merge tree remains to be executed. Also, two arrays are sorted in tandem,
-   * as discussed above.
+   * merge tree remains to be executed. The two arrays are sorted in tandem.
    * @param <T> the data type
    * @param quantiles array of quantiles
    * @param cumWts array of cum weights

src/main/java/org/apache/datasketches/quantiles/UpdatableQuantilesDoublesSketch.java

@@ -24,7 +24,7 @@
 import org.apache.datasketches.common.MemorySegmentRequest;
 
 /**
- * Extends QuantilesDoubleSketch
+ * Extends QuantilesDoublesSketch
  * @author Jon Malkin
  */
 public abstract class UpdatableQuantilesDoublesSketch extends QuantilesDoublesSketch {

src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java

@@ -155,12 +155,12 @@ public static DoublesPair includeDoublesMinMax(
   }
 
   /**
-   * The logic to include the min and max of type double.
+   * The logic to include the min and max of type long.
    * @param quantilesIn The array of quantiles
    * @param cumWeightsIn The array of associated cumulative weights
    * @param maxItem the maximum item of the stream
    * @param minItem the minimum item of the stream
-   * @return a DoublesPair
+   * @return a LongsPair
    */
   public static LongsPair includeLongsMinMax(
           final long[] quantilesIn,