diff --git a/pom.xml b/pom.xml index f518c905f..4aef542c0 100644 --- a/pom.xml +++ b/pom.xml @@ -135,10 +135,16 @@ under the License. 7.12.0 2.0.12 - generate_java_files - check_cpp_files - check_go_files - check_cpp_historical_files + generate_java_files + check_java_files + check_cpp_files + check_go_files + check_rust_files + check_cpp_historical_files + ${testng.generate_java_files},${testng.check_java_files},${testng.check_cpp_files},${testng.check_go_files},${testng.check_rust_files},${testng.check_cpp_historical_files} + + ${project.basedir} + ${project.basedir} -Xmx4g @@ -373,10 +379,21 @@ under the License. ${maven-javadoc-plugin.version} en_US + ${project.basedir}/src/main/javadoc/overview.html ${project.reporting.outputDirectory} true public - none + + all true -J${jvm.mem} @@ -412,6 +429,22 @@ under the License. package jar + + + validate-internal-javadoc + verify + javadoc + + private + all,-missing + ${project.build.directory}/javadoc-internal-check + + @@ -471,15 +504,22 @@ under the License. false true false - true + false ${project.build.directory}/test-output/${maven.build.timestamp} false + + ${project.basedir}/src/test/resources/testng.xml + + ${testng.all_groups} - verbose - 0 + verbose0 + + ${test.data.root} + ${project.root} + @@ -720,15 +760,63 @@ under the License. - generate-java-files + generate_java_files + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testng.generate_java_files} + + + + + + + + + check_all_language_files + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testng.all_groups} + + + + + + + + + check_java_files + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${testng.generate_java_files},${testng.check_java_files} + + + + + + + + + check_cpp_files org.apache.maven.plugins maven-surefire-plugin - ${testng.generate-java-files} - ${testng.check-cpp-files},${testng.check-go-files},${testng.check-cpp-historical-files} + ${testng.check_cpp_files} + @@ -736,15 +824,15 @@ under the License. - check-cpp-files + check_go_files org.apache.maven.plugins maven-surefire-plugin - ${testng.check-cpp-files} - ${testng.generate-java-files},${testng.check-go-files},${testng.check-cpp-historical-files} + ${testng.check_go_files} + @@ -752,15 +840,15 @@ under the License. - check-go-files + check_rust_files org.apache.maven.plugins maven-surefire-plugin - ${testng.check-go-files} - ${testng.generate-java-files},${testng.check-cpp-files},${testng.check-cpp-historical-files} + ${testng.check_rust_files} + @@ -768,15 +856,15 @@ under the License. - check-cpp-historical-files + check_cpp_historical_files org.apache.maven.plugins maven-surefire-plugin - ${testng.check-cpp-historical-files} - ${testng.generate-java-files},${testng.check-go-files},${testng.check-cpp-files} + ${testng.check_cpp_historical_files} + diff --git a/src/main/java/org/apache/datasketches/common/ArrayOfStringsSerDe.java b/src/main/java/org/apache/datasketches/common/ArrayOfStringsSerDe.java index 72e2ebea4..9375ad07b 100644 --- a/src/main/java/org/apache/datasketches/common/ArrayOfStringsSerDe.java +++ b/src/main/java/org/apache/datasketches/common/ArrayOfStringsSerDe.java @@ -35,8 +35,6 @@ * this method is 2 times more compact, but it takes more time to encode and decode * by a factor of 1.5 to 2. * - *

The serialization - * * @author Alexander Saydakov */ public class ArrayOfStringsSerDe extends ArrayOfItemsSerDe { diff --git a/src/main/java/org/apache/datasketches/common/BoundsOnRatiosInSampledSets.java b/src/main/java/org/apache/datasketches/common/BoundsOnRatiosInSampledSets.java index b65afe43a..acb4f63c7 100644 --- a/src/main/java/org/apache/datasketches/common/BoundsOnRatiosInSampledSets.java +++ b/src/main/java/org/apache/datasketches/common/BoundsOnRatiosInSampledSets.java @@ -47,7 +47,7 @@ private BoundsOnRatiosInSampledSets() {} * @param f the inclusion probability used to produce the set with size a and should * generally be less than 0.5. Above this value, the results not be reliable. * When f = 1.0 this returns the estimate. - * @return the approximate upper bound + * @return the approximate lower bound */ public static double getLowerBoundForBoverA(final long a, final long b, final double f) { checkInputs(a, b, f); @@ -61,7 +61,7 @@ public static double getLowerBoundForBoverA(final long a, final long b, final do * @param a See class javadoc * @param b See class javadoc * @param f the inclusion probability used to produce the set with size a. - * @return the approximate lower bound + * @return the approximate upper bound */ public static double getUpperBoundForBoverA(final long a, final long b, final double f) { checkInputs(a, b, f); @@ -86,7 +86,7 @@ public static double getEstimateOfBoverA(final long a, final long b) { * Return the estimate of A. See class javadoc. * @param a See class javadoc * @param f the inclusion probability used to produce the set with size a. - * @return the approximate lower bound + * @return the approximate estimate of A. */ public static double getEstimateOfA(final long a, final double f) { checkInputs(a, 1, f); @@ -97,7 +97,7 @@ public static double getEstimateOfA(final long a, final double f) { * Return the estimate of B. See class javadoc. * @param b See class javadoc * @param f the inclusion probability used to produce the set with size b. - * @return the approximate lower bound + * @return the approximate estimate of B. */ public static double getEstimateOfB(final long b, final double f) { checkInputs(b + 1, b, f); diff --git a/src/main/java/org/apache/datasketches/common/MemorySegmentRequestExample.java b/src/main/java/org/apache/datasketches/common/MemorySegmentRequestExample.java index fbf2521b8..3a45bb85b 100644 --- a/src/main/java/org/apache/datasketches/common/MemorySegmentRequestExample.java +++ b/src/main/java/org/apache/datasketches/common/MemorySegmentRequestExample.java @@ -40,7 +40,7 @@ public final class MemorySegmentRequestExample implements MemorySegmentRequest { public MemorySegmentRequestExample() { } /** - * Request a new off-heap MemorySegment with the given newByteSeze. + * Request a new off-heap MemorySegment with the given newByteSize. * An internal confined Arena is created to exclusively manage the new segment and it is associated * with the new segment with a ConcurrentHashMap. */ diff --git a/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java b/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java index 62727ebed..4e060e2b1 100644 --- a/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java +++ b/src/main/java/org/apache/datasketches/hll/AbstractCoupons.java @@ -51,8 +51,8 @@ abstract class AbstractCoupons extends HllSketchImpl { *

Note: This is an approximation to the true mapping from numCoupons to N, * which has a range of validity roughly from 0 to 6 million coupons.

* - *

The k of the implied coupon sketch, which must not be confused with the k of the HLL - * sketch. In this application k is always 2^26, which is the number of address bits of the + *

The k of the implied coupon sketch must not be confused with the k of the HLL + * sketch. In this context k is always 2^26, which is the number of address bits of the * 32-bit coupon.

* @return the unique count estimate. */ diff --git a/src/main/java/org/apache/datasketches/hll/HllSketch.java b/src/main/java/org/apache/datasketches/hll/HllSketch.java index 0ff0c1e97..b1e958651 100644 --- a/src/main/java/org/apache/datasketches/hll/HllSketch.java +++ b/src/main/java/org/apache/datasketches/hll/HllSketch.java @@ -81,8 +81,8 @@ * *
{@code
  * double estimate = sketch.getEstimate();
- * double estUB = sketch.getUpperBound(1.0); //the upper bound at 1 standard deviation.
- * double estLB = sketch.getLowerBound(1.0); //the lower bound at 1 standard deviation.
+ * double estUB = sketch.getUpperBound(1); //the upper bound at 1 standard deviation.
+ * double estLB = sketch.getLowerBound(1); //the lower bound at 1 standard deviation.
  * //OR
  * System.out.println(sketch.toString()); //will output a summary of the sketch.
  * }
diff --git a/src/main/java/org/apache/datasketches/hll/PairIterator.java b/src/main/java/org/apache/datasketches/hll/PairIterator.java index 0282152a3..1e56ec7f8 100644 --- a/src/main/java/org/apache/datasketches/hll/PairIterator.java +++ b/src/main/java/org/apache/datasketches/hll/PairIterator.java @@ -36,13 +36,14 @@ String getHeader() { * In LIST and SET modes, this gets the iterating index into the integer array of HLL key/value * pairs. * In HLL mode, this is the iterating index into the hypothetical array of HLL values, which may - * be physically contructed differently based on the compaction scheme (HLL_4, HLL_6, HLL_8). + * be physically constructed differently based on the compaction scheme (HLL_4, HLL_6, HLL_8). * @return the index. */ abstract int getIndex(); /** - * Gets the key, the low 26 bits of an pair, and can be up to 26 bits in length. + * Gets the key from the low 26 bits of a pair. + * The key corresponds to the address in the HLL array and can be up to 26 bits in length. * @return the key */ abstract int getKey(); diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java index 0283c015b..6638007b6 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDirectDoublesSketch.java @@ -86,7 +86,7 @@ class KllDirectDoublesSketch extends KllDoublesSketch { * @param k parameter that controls size of the sketch and accuracy of estimates * @param m parameter that controls the minimum level width in items. * @param dstSeg the given destination MemorySegment object for use by the sketch - * @param mSegReq the callback for the sketch to request a larger MemorySegment. + * @param mSegmentRequest the callback for the sketch to request a larger MemorySegment. * @return a new instance of this sketch */ static KllDirectDoublesSketch newDirectUpdatableInstance( diff --git a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java index 1de0b2ff1..d4c89322b 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDirectLongsSketch.java @@ -86,7 +86,7 @@ class KllDirectLongsSketch extends KllLongsSketch { * @param k parameter that controls size of the sketch and accuracy of estimates * @param m parameter that controls the minimum level width in items. * @param dstSeg the given destination MemorySegment object for use by the sketch - * @param mSegReq the callback for the sketch to request a larger MemorySegment. + * @param mSegmentRequest the callback for the sketch to request a larger MemorySegment. * @return a new instance of this sketch */ static KllDirectLongsSketch newDirectUpdatableInstance( diff --git a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java index e7df5cfd6..27a6372bb 100644 --- a/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java +++ b/src/main/java/org/apache/datasketches/kll/KllPreambleUtil.java @@ -52,7 +52,7 @@ * This allows the possibility of the introduction of different serialization * schemes with minimal impact on the rest of the library.

* - *

Visual Layout

+ *

Visual Layout

* The low significance bytes of the visual data structure below are on the left. * The multi-byte primitives are stored in native byte order. * The numeric byte and short fields are treated as unsigned. diff --git a/src/main/java/org/apache/datasketches/quantiles/CompactQuantilesDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/CompactQuantilesDoublesSketch.java index 490ac35eb..f7d304fc0 100644 --- a/src/main/java/org/apache/datasketches/quantiles/CompactQuantilesDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/CompactQuantilesDoublesSketch.java @@ -37,7 +37,7 @@ boolean isCompact() { @Override public boolean isReadOnly() { - return false; + return true; } @Override diff --git a/src/main/java/org/apache/datasketches/quantiles/DoublesUtil.java b/src/main/java/org/apache/datasketches/quantiles/DoublesUtil.java index da6b89579..4be15d55f 100644 --- a/src/main/java/org/apache/datasketches/quantiles/DoublesUtil.java +++ b/src/main/java/org/apache/datasketches/quantiles/DoublesUtil.java @@ -34,9 +34,7 @@ /** * Utilities that support the doubles quantiles algorithms. * - *

This class contains a highly specialized sort called blockyTandemMergeSort(). - * It also contains methods that are used while building histograms and other common - * functions.

+ *

This class contains methods that are used while building histograms and other common functions.

* * @author Lee Rhodes */ diff --git a/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java b/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java index 5105abe8a..84d62dee6 100644 --- a/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java +++ b/src/main/java/org/apache/datasketches/quantiles/ItemsMergeImpl.java @@ -219,8 +219,7 @@ private static void justZipWithStride( * blockyTandemMergeSort() is an implementation of top-down merge sort specialized * for the case where the input contains successive equal-length blocks * that have already been sorted, so that only the top part of the - * merge tree remains to be executed. Also, two arrays are sorted in tandem, - * as discussed above. + * merge tree remains to be executed. The two arrays are sorted in tandem. * @param the data type * @param quantiles array of quantiles * @param cumWts array of cum weights diff --git a/src/main/java/org/apache/datasketches/quantiles/UpdatableQuantilesDoublesSketch.java b/src/main/java/org/apache/datasketches/quantiles/UpdatableQuantilesDoublesSketch.java index 803484aac..1294d0de0 100644 --- a/src/main/java/org/apache/datasketches/quantiles/UpdatableQuantilesDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/quantiles/UpdatableQuantilesDoublesSketch.java @@ -24,7 +24,7 @@ import org.apache.datasketches.common.MemorySegmentRequest; /** - * Extends QuantilesDoubleSketch + * Extends QuantilesDoublesSketch * @author Jon Malkin */ public abstract class UpdatableQuantilesDoublesSketch extends QuantilesDoublesSketch { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java index 84061f840..f3648cec6 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/IncludeMinMax.java @@ -155,12 +155,12 @@ public static DoublesPair includeDoublesMinMax( } /** - * The logic to include the min and max of type double. + * The logic to include the min and max of type long. * @param quantilesIn The array of quantiles * @param cumWeightsIn The array of associated cumulative weights * @param maxItem the maximum item of the stream * @param minItem the minimum item of the stream - * @return a DoublesPair + * @return a LongsPair */ public static LongsPair includeLongsMinMax( final long[] quantilesIn, diff --git a/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java b/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java index 52d93362e..8215625fb 100644 --- a/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/sampling/PreambleUtil.java @@ -39,9 +39,9 @@ /** * This class defines the preamble items structure and provides basic utilities for some of the key * fields. Fields are presented in Little Endian format, but multi-byte values (int, long, double) - * are stored in native byte order. All byte values are treated as unsigned. + * are stored in native byte order. All byte values are treated as unsigned. * - *

Reservoir Sampling

+ *

Reservoir Sampling

* *

Sketch: The count of items seen is limited to 48 bits (~256 trillion) even * though there are adjacent unused preamble bits. The acceptance probability for an item is a @@ -80,7 +80,7 @@ * * * - *

VarOpt Sampling

+ *

VarOpt Sampling

* *

VarOpt: A VarOpt sketch has a more complex internal items structure and * requires a larger preamble. Values serving a similar purpose in both reservoir and varopt sampling @@ -129,7 +129,7 @@ * * * - *

EPPS Sampling

+ *

EPPS Sampling

* *

An empty sketch requires 8 bytes. * diff --git a/src/main/java/org/apache/datasketches/sampling/VarOptItemsSamples.java b/src/main/java/org/apache/datasketches/sampling/VarOptItemsSamples.java index 00ad3fd6f..15d84512b 100644 --- a/src/main/java/org/apache/datasketches/sampling/VarOptItemsSamples.java +++ b/src/main/java/org/apache/datasketches/sampling/VarOptItemsSamples.java @@ -217,12 +217,9 @@ public void setClass(final Class clazz) { } /** - * Returns the length Copies items and weights from the sketch, if necessary, and returns the - * length of - * any - * resulting array. The result will be 0 for an empty sketch. + * Returns the number of samples from the sketch. The result will be 0 for an empty sketch. * - * @return The number of items and weights in the sketch + * @return The number of samples (items and weights) in the sketch */ public int getNumSamples() { loadArrays(); @@ -232,7 +229,7 @@ public int getNumSamples() { /** * Returns a shallow copy of the array of sample items contained in the sketch. If this is the * first getter call, copies data arrays from the sketch. - * @return The number of samples contained in the sketch. + * @return The samples contained in the sketch. */ public T[] items() { loadArrays(); @@ -253,7 +250,7 @@ public T items(final int i) { /** * Returns a copy of the array of weights contained in the sketch. If this is the first * getter call, copies data arrays from the sketch. - * @return The number of samples contained in the sketch. + * @return The array of weights contained in the sketch. */ public double[] weights() { loadArrays(); diff --git a/src/main/java/org/apache/datasketches/tdigest/BinarySearch.java b/src/main/java/org/apache/datasketches/tdigest/BinarySearch.java index cdd14b89b..209d1727e 100644 --- a/src/main/java/org/apache/datasketches/tdigest/BinarySearch.java +++ b/src/main/java/org/apache/datasketches/tdigest/BinarySearch.java @@ -31,9 +31,9 @@ public BinarySearch() { } /** * Returns an index to the first element in the range [first, last) such that - * element < value is false (i.e. that is greater than or equal to value), + * {@code element < value} is false (i.e. that is greater than or equal to value), * or last if no such element is found. - * The range [first, last) must be partitioned with respect to the expression element < value, + * The range [first, last) must be partitioned with respect to the expression {@code element < value}, * i.e., all elements for which the expression is true must precede all elements * for which the expression is false. * A fully-sorted range meets this criterion. @@ -64,9 +64,9 @@ static int lowerBound(final double[] values, int first, final int last, final do /** * Returns an index to the first element in the range [first, last) such that - * value < element is true (i.e. that is strictly greater than value), + * {@code value < element} is true (i.e. that is strictly greater than value), * or last if no such element is found. - * The range [first, last) must be partitioned with respect to the expression !(value < element), + * The range [first, last) must be partitioned with respect to the expression {@code !(value < element)}, * i.e., all elements for which the expression is true must precede all elements * for which the expression is false. * A fully-sorted range meets this criterion. diff --git a/src/main/java/org/apache/datasketches/theta/CompactOperations.java b/src/main/java/org/apache/datasketches/theta/CompactOperations.java index 46d2d7e7e..99d406600 100644 --- a/src/main/java/org/apache/datasketches/theta/CompactOperations.java +++ b/src/main/java/org/apache/datasketches/theta/CompactOperations.java @@ -326,7 +326,7 @@ static long[] compactCache(final long[] srcCache, final int curCount, /** * This corrects a temporary anomalous condition where compact() or toByteArray() is called on an UpdatableThetaSketch - * that was initialized with p < 1.0 and update() was never called. In this case Theta < 1.0, + * that was initialized with {@code p < 1.0} and update() was never called. In this case {@code Theta < 1.0}, * curCount = 0, and empty = true. The correction is to change Theta to 1.0, which makes the * returning sketch empty. This should only be used in the compaction or serialization of an * UpdatableThetaSketch. diff --git a/src/main/java/org/apache/datasketches/theta/ThetaSketch.java b/src/main/java/org/apache/datasketches/theta/ThetaSketch.java index 1b82ec76b..8dcefacdc 100644 --- a/src/main/java/org/apache/datasketches/theta/ThetaSketch.java +++ b/src/main/java/org/apache/datasketches/theta/ThetaSketch.java @@ -551,7 +551,7 @@ public static String toString(final MemorySegment seg) { * Gets the internal cache array. For on-heap sketches this will return a reference to the actual * cache array. For MemorySegment-based sketches this returns a copy. * - *

This can be an expensive operation and is intended for diagnostic & test applications. + *

This can be an expensive operation and is intended for diagnostic & test applications. * Use {@link #iterator() iterator()} instead.

* @return the internal cache array. */ diff --git a/src/main/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSets.java b/src/main/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSets.java index 5337efc44..36687b1b0 100644 --- a/src/main/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSets.java +++ b/src/main/java/org/apache/datasketches/thetacommon/BoundsOnRatiosInThetaSketchedSets.java @@ -39,9 +39,9 @@ *
  • The Lower Bound estimate on the ratio PopB/PopA is * BoundsOnRatiosInThetaSketchedSets.getLowerBoundForBoverA(A, B).
  • * - * Note: The theta of A cannot be greater than the theta of B. + * Note: The theta of B cannot be greater than the theta of A. * If B is formed as an intersection of A and some other set C, - * then the theta of B is guaranteed to be less than or equal to the theta of B. + * then the theta of B is guaranteed to be less than or equal to the theta of A. * * @author Kevin Lang * @author Lee Rhodes diff --git a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java index e7268ffef..e0922ff1f 100644 --- a/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java +++ b/src/main/java/org/apache/datasketches/tuple/adouble/DoubleSummary.java @@ -32,7 +32,7 @@ * Summary for generic tuple sketches of type Double. * This summary keeps a double value. On update a predefined operation is performed depending on * the mode. - * Supported modes: Sum, Min, Max, AlwaysOne, Increment. The default mode is Sum. + * Supported modes: Sum, Min, Max, AlwaysOne. */ public final class DoubleSummary implements UpdatableSummary { private double value_; diff --git a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java index b8c7c23fb..60b3221d6 100644 --- a/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java +++ b/src/main/java/org/apache/datasketches/tuple/aninteger/IntegerSummary.java @@ -84,7 +84,7 @@ private IntegerSummary(final int value, final Mode mode) { /** * Creates an instance of IntegerSummary with a given mode. - * @param mode update mode. This should not be called by a user. + * @param mode update mode. */ public IntegerSummary(final Mode mode) { mode_ = mode; diff --git a/src/main/java/org/apache/datasketches/tuple/strings/package-info.java b/src/main/java/org/apache/datasketches/tuple/strings/package-info.java index 9085103da..aef98afa1 100644 --- a/src/main/java/org/apache/datasketches/tuple/strings/package-info.java +++ b/src/main/java/org/apache/datasketches/tuple/strings/package-info.java @@ -18,7 +18,7 @@ */ /** - * This package is for a generic implementation of the Tuple sketch for single String value. + * This package is for a generic implementation of the Tuple sketch for String arrays. */ package org.apache.datasketches.tuple.strings; diff --git a/src/main/javadoc/overview.html b/src/main/javadoc/overview.html index 9124aa98a..536f0f16a 100644 --- a/src/main/javadoc/overview.html +++ b/src/main/javadoc/overview.html @@ -23,7 +23,7 @@ -

    Sketching Core Library

    +

    Sketching Core Library For Java

    Overview

    The Sketching Core Library provides a range of stochastic streaming algorithms and closely @@ -33,9 +33,9 @@

    Overview

    This library is divided into packages that constitute distinct groups of functionality:

    -Note: In general, if the requirements or promises of any method's contract are not fulfilled +

    Note: In general, if the requirements or promises of any method's contract are not fulfilled (that is, if there is a bug in either the method or its caller), then an unchecked exception will be thrown. -The precise type of such an unchecked exception does not form part of any method's contract. +The precise type of such an unchecked exception does not form part of any method's contract.

    diff --git a/src/test/java/org/apache/datasketches/common/A_BeforeSuite.java b/src/test/java/org/apache/datasketches/common/A_BeforeSuite.java index 97fe267ff..038b78f14 100644 --- a/src/test/java/org/apache/datasketches/common/A_BeforeSuite.java +++ b/src/test/java/org/apache/datasketches/common/A_BeforeSuite.java @@ -19,15 +19,36 @@ package org.apache.datasketches.common; +import java.util.concurrent.atomic.AtomicInteger; + import org.testng.annotations.BeforeSuite; public class A_BeforeSuite { + /** + * For counting cross-language test cases. + */ + private final AtomicInteger count = new AtomicInteger(0); + + public void incCount() { + count.incrementAndGet(); // getAndIncrement() + } + + public int getCount() { + return count.get(); + } + + public void resetCount() { + count.set(0); + } + @BeforeSuite(alwaysRun = true) public void printTestEnvironment() { System.out.println("===================================================="); System.out.println("TEST JDK: " + System.getProperty("java.version")); System.out.println("TEST JDK HOME: " + System.getProperty("java.home")); System.out.println("====================================================="); + resetCount(); } + } diff --git a/src/test/java/org/apache/datasketches/common/TestUtil.java b/src/test/java/org/apache/datasketches/common/TestUtil.java deleted file mode 100644 index fbe39a856..000000000 --- a/src/test/java/org/apache/datasketches/common/TestUtil.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.common; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Objects; - -/** - * Utilities common to testing - */ -public final class TestUtil { - - private static final String userDir = System.getProperty("user.dir"); - - /** - * TestNG group constants - */ - public static final String GENERATE_JAVA_FILES = "generate_java_files"; - public static final String CHECK_CPP_FILES = "check_cpp_files"; - public static final String CHECK_GO_FILES = "check_go_files"; - public static final String CHECK_RUST_FILES = "check_rust_files"; - public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files"; - - /** - * The project relative Path for Java serialized sketches to be tested by other languages. - */ - public static final Path javaPath = Path.of(userDir, "serialization_test_data", "java_generated_files").normalize(); - - /** - * The project relative Path for C++ serialized sketches to be tested by Java. - */ - public static final Path cppPath = Path.of(userDir, "serialization_test_data", "cpp_generated_files").normalize(); - - /** - * The project relative Path for Go serialized sketches to be tested by Java. - */ - public static final Path goPath = Path.of(userDir, "serialization_test_data", "go_generated_files").normalize(); - - /** - * The project relative Path for Rust serialized sketches to be tested by Java. - */ - public static final Path rustPath = Path.of(userDir, "serialization_test_data", "rust_generated_files").normalize(); - - /** - * The project relative Path for /src/test/resources - */ - public static final Path resPath = Path.of(userDir,"src","test","resources").normalize(); - - public enum Existence { MUST_EXIST, WARNING } - - /** - * Gets all the bytes of a file as a byte array. - * If the file is missing, this either throws an exception or writes a warning message to the console - * based on the state of the optional {@link #Existence Existence}. - * @param basePath the base directory path where the file is located - * @param fileName the simple file name of the file - * @param option an optional parameter. If option == Existence.MUST_EXIST and the file does not exist an exception will be thrown. - * If option == Existence.WARNING, or not given, and the file does not exist, it writes a warning message - * to {@link System.err System.err}. - * If option has more than one argument an exception will be thrown. - * @return a byte array. It may be empty. - * @throws RuntimeException for IO errors, or if resolved path is not a file or not readable or optionally not found. - */ - public static byte[] getFileBytes(final Path basePath, final String fileName, Existence... option) { - Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); - Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); - if (option.length > 1) { throw new IllegalArgumentException("Existence option has a maximum of one argument"); } - Existence status = (option.length == 1) ? option[0] : Existence.WARNING; - - Path path = Path.of(basePath.toString(), fileName); - Path absPath = path.toAbsolutePath(); //for error output - if (Files.notExists(path)) { - if (status == Existence.MUST_EXIST) { - throw new RuntimeException("File disappeared or not found: " + absPath); - } else { - System.err.println("WARNING: File disappeared or not found: " + absPath); - return new byte[0]; - } - } - if (!Files.isRegularFile(path) || !Files.isReadable(path)) { - throw new RuntimeException("Path is not a regular file or not readable: " + absPath); - } - try { - byte[] bytes = Files.readAllBytes(path); - return bytes; - } catch (IOException e) { - throw new RuntimeException("System IO Error reading file: " + absPath + " " + e); - } - } - - /** - * Puts all the bytes of the given byte array to a file with the given fileName. - * This assumes that the base directory path is {@link #javaPath javaPath}. - * @param fileName the name of the target file - * @param bytes the given byte array - */ - public static void putBytesToJavaPath(final String fileName, final byte[] bytes) { - putBytesToFile(javaPath, fileName, bytes); - } - - /** - * Puts all the bytes of the given byte array to a basePath file with the given fileName. - * If the file exists it will be overwritten. - * @param basePath the directory path for the given fileName - * @param fileName the name of the target file - * @param bytes the given byte array - * @throws RuntimeException for IO errors, - */ - public static void putBytesToFile(final Path basePath, final String fileName, final byte[] bytes) { - Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); - Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); - Objects.requireNonNull(bytes, "input parameter 'byte[] bytes' cannot be null."); - Path filePath = null; - try { - Files.createDirectories(basePath); //create the directory if it doesn't exist. - filePath = basePath.resolve(fileName); - Files.write(filePath, bytes); - } catch (IOException e) { - final String filePathDesc = String.valueOf(filePath); - throw new RuntimeException("System IO Error writing file: " + filePathDesc + " " + e); - } - } -} diff --git a/src/test/java/org/apache/datasketches/common/TestUtilTest.java b/src/test/java/org/apache/datasketches/common/TestUtilTest.java deleted file mode 100644 index e8051fe88..000000000 --- a/src/test/java/org/apache/datasketches/common/TestUtilTest.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.datasketches.common; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToFile; -import static org.apache.datasketches.common.TestUtil.resPath; -import static org.apache.datasketches.common.TestUtil.Existence.MUST_EXIST; -import static org.apache.datasketches.common.TestUtil.Existence.WARNING; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; - -import java.io.IOException; -import java.nio.file.Path; - -import org.testng.annotations.Test; - -public class TestUtilTest { - - @Test - public void testGetFileBytes_Success() {// throws IOException { - byte[] resultBytes = getFileBytes(resPath, "GettysburgAddress.txt"); - assertNotNull(resultBytes); - String resultString = new String(resultBytes, UTF_8); - assertTrue(resultString.startsWith("Abraham Lincoln's Gettysburg Address:")); - } - - @Test - public void testGetFileBytes_MissingFile_Warning() { - byte[] resultBytes = getFileBytes(resPath, "Test_NonExistentFile_OK", WARNING); //WARNING is the default - assertNotNull(resultBytes); - assertEquals(resultBytes.length, 0, "Should return empty array for missing file."); - } - - @Test(expectedExceptions = RuntimeException.class) - public void testGetFileBytes_MissingFile_MustExist() { - getFileBytes(resPath, "Test_NonExistentFile_OK", MUST_EXIST); - } - - @Test(expectedExceptions = RuntimeException.class) - public void testGetFileBytes_NotRegular_NotReadable() throws IOException { - getFileBytes(resPath, ""); - } - - private static final Path testPath = Path.of(".", "target", "testDir"); - - @Test - public void testPutBytesToFile() { - byte[] gettysBytes = getFileBytes(resPath, "GettysburgAddress.txt"); - putBytesToFile(testPath, "GettysburgAddressCopy.txt", gettysBytes); - byte[] gettysBytes2 = getFileBytes(testPath, "GettysburgAddressCopy.txt"); - assertEquals(gettysBytes, gettysBytes2); - } - -} diff --git a/src/test/java/org/apache/datasketches/common/UtilTest.java b/src/test/java/org/apache/datasketches/common/UtilTest.java index da8858ca7..f9f0b09ae 100644 --- a/src/test/java/org/apache/datasketches/common/UtilTest.java +++ b/src/test/java/org/apache/datasketches/common/UtilTest.java @@ -20,8 +20,6 @@ package org.apache.datasketches.common; import static java.lang.Math.pow; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.javaPath; import static org.apache.datasketches.common.Util.bytesToInt; import static org.apache.datasketches.common.Util.bytesToLong; import static org.apache.datasketches.common.Util.bytesToString; @@ -56,7 +54,6 @@ import static org.apache.datasketches.common.Util.zeroPad; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -436,11 +433,7 @@ static void checkCeilingMultiple2expK() { assertEquals(v, v2); } - @Test - public void checkDirCreation() { - assertNotNull(javaPath); - assertNotNull(cppPath); - } + @Test public void printlnTest() { diff --git a/src/test/java/org/apache/datasketches/common/UtilityIO.java b/src/test/java/org/apache/datasketches/common/UtilityIO.java new file mode 100644 index 000000000..3128ab6ab --- /dev/null +++ b/src/test/java/org/apache/datasketches/common/UtilityIO.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Objects; + +/** + * Utilities common to testing + */ +public final class UtilityIO { + private static final String LS = System.getProperty("line.separator"); + private static final Class clazz = UtilityIO.class; + private static final ClassLoader CL = clazz.getClassLoader(); + private static final String TEST_DATA_ROOT_PROP = "test.data.root"; + private static final String PROJECT_ROOT_PROP = "project.root"; + + /** + * TestNG group constants + */ + public static final String GENERATE_JAVA_FILES = "generate_java_files"; + public static final String CHECK_JAVA_FILES = "check_java_files"; + public static final String CHECK_CPP_FILES = "check_cpp_files"; + public static final String CHECK_GO_FILES = "check_go_files"; + public static final String CHECK_RUST_FILES = "check_rust_files"; + public static final String CHECK_CPP_HISTORICAL_FILES = "check_cpp_historical_files"; + + public enum Existence { MUST_EXIST, WARNING } + + /** + * For passing variables to a multi-language test method leveraging TestNG Groups. + * + *

    Usage in a method would look like this:

    + * {@snippet : + * public void testMethod(GpLanguage lang) { + * String suffix = lang.sfx; + * Path path = lang.pth; + * ... + * } + * } + */ + public enum GroupLanguage { + JAVA("_java", getJavaPath()), + CPP("_cpp", getCppPath()), + GO("_go", getGoPath()), + RUST("_rust", getRustPath()); + + + public final String sfx; + public final Path pth; + + GroupLanguage(String suffix, Path path) { + this.sfx = suffix; + this.pth = path; + } + } + + /** + * The project relative Path for Java serialized sketches to be tested by other languages. + */ + public static Path getJavaPath() { + return getTestDataRoot().resolve("serialization_test_data", "java_generated_files"); + } + + /** + * The project relative Path for C++ serialized sketches to be tested by Java. + */ + public static Path getCppPath() { + return getTestDataRoot().resolve("serialization_test_data", "cpp_generated_files"); + } + + /** + * The project relative Path for Go serialized sketches to be tested by Java. + */ + public static Path getGoPath() { + return getTestDataRoot().resolve("serialization_test_data", "go_generated_files"); + } + + /** + * The project relative Path for Rust serialized sketches to be tested by Java. + */ + public static Path getRustPath() { + return getTestDataRoot().resolve("serialization_test_data", "rust_generated_files"); + } + + /** + * Gets all the bytes of a file as a byte array. + * If the file is missing this issues a Warning message to the console. + * @param basePath the directory where fileName is located. + * @param fileName the simple file name of the file. + * @return a byte array. It will be empty if file not found. + * @throws RuntimeException for IO errors. + */ + public static byte[] getFileBytes(final Path basePath, final String fileName) { + return getFileBytes(basePath, fileName, Existence.WARNING); + } + + /** + * Gets all the bytes of a file as a byte array. + * If the file is missing this issues a Warning message to the console or throws a RuntimeException + * based on the state of Existence. + * @param basePath the directory where fileName is located. + * @param fileName the simple file name of the file. + * @return a byte array. It will be empty if file not found and Existence is WARNING + * @throws RuntimeException for IO errors, + */ + public static byte[] getFileBytes(final Path basePath, final String fileName, final Existence existence) { + Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); + Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); + Objects.requireNonNull(existence, "input parameter 'Existence existence' cannot be null."); + + Path path = basePath.resolve(fileName).normalize(); + String pathDisplay; + try { + pathDisplay = path.toAbsolutePath().toString(); + } catch (SecurityException e) { + pathDisplay = path.toString(); + } + + try { + return Files.readAllBytes(path); + } catch (NoSuchFileException e) { + if (existence == Existence.MUST_EXIST) { + throw new RuntimeException("File not found: " + pathDisplay, e); + } + System.err.println("WARNING: File not found: " + pathDisplay); + return new byte[0]; + } catch (AccessDeniedException e) { + throw new RuntimeException("Permission denied reading file: " + pathDisplay, e); + } catch (IOException e) { + throw new RuntimeException("IO Error reading file: " + pathDisplay, e); + } + } + + /** + * This gets a byte[] from a resource file in /src/test/resources whether called on an install or a jar. + * These files must exist. + * @param fileName the desired filename + * @return the bytes of the file as a byte[] + * @throws IllegalArgumentException if file not found. + */ + public static byte[] getTestResourceBytes(final String fileName) { + Objects.requireNonNull(fileName, "fileName cannot be null"); + // Strip leading slash if present, as ClassLoader doesn't accept leading slashes + String resourcePath = fileName.startsWith("/") ? fileName.substring(1) : fileName; + try (InputStream is = CL.getResourceAsStream(resourcePath)) { + if (is == null) { + throw new IllegalArgumentException("Resource not found on classpath: " + resourcePath); + } + return is.readAllBytes(); + } catch (final IOException e){ + throw new RuntimeException("Error reading resource: " + resourcePath, e); + } + } + + /** + * Puts all the bytes of the given byte array to a file with the given fileName. + * This assumes that the base directory path is {@link #javaPath javaPath}. + * @param fileName the name of the target file + * @param bytes the given byte array + */ + public static void putBytesToJavaPath(final String fileName, final byte[] bytes) { + putBytesToFile(getJavaPath(), fileName, bytes); + } + + /** + * Puts all the bytes of the given byte array to a basePath file with the given fileName. + * If the file exists it will be overwritten. Called from putBytesToJavaPath() and TestUtilTest. + * + *

    Used by putBytesToJavaPath(...) + * @param basePath the directory path for the given fileName + * @param fileName the name of the target file + * @param bytes the given byte array + * @throws RuntimeException for IO errors, + */ + static void putBytesToFile(final Path basePath, final String fileName, final byte[] bytes) { + Objects.requireNonNull(basePath, "input parameter 'Path basePath' cannot be null."); + Objects.requireNonNull(fileName, "input parameter 'String fileName' cannot be null."); + Objects.requireNonNull(bytes, "input parameter 'byte[] bytes' cannot be null."); + Path filePath = null; + try { + Files.createDirectories(basePath); //create the directory if it doesn't exist. + filePath = basePath.resolve(fileName); + Files.write(filePath, bytes); + } catch (IOException e) { + final String filePathDesc = String.valueOf(filePath); + throw new RuntimeException("System IO Error writing file: " + filePathDesc + " " + e); + } + } + + /** + * Resolves the target directory for test data generation/reading. + * The default is the project root, but it can be overridden like this: + * + *

    mvn clean test -Pcheck-cpp-files -Dtest.data.root="/tmp/custom_path

    + * + *

    Calling the binary test jar: see src/test/resources/testng.xml

    + * + * Used by getJavaPath(), getCppPath(), getGoPath(), getRustPath() + * + * @return Path to the test.data.root directory. + * @throws IllegalStateException if running from a JAR without -Dtest.data.root defined. + */ + private static Path getTestDataRoot() { + // Explicit user override or Maven-injected property + String sysProp = System.getProperty(TEST_DATA_ROOT_PROP); + if (sysProp != null && !sysProp.isBlank()) { + return Path.of(sysProp).toAbsolutePath().normalize(); + } + + checkExecutionFromJar(TEST_DATA_ROOT_PROP); + + // Fallback for local source checkout / IDE runs (walk up to top-level .mvn) + return getProjectRoot(); + } + + /** + * Resolves the project root + * The default is the project root, but it can be overridden like this: + * + *

    mvn clean test -Dproject.root="/tmp/custom_path

    + * + *

    Calling the binary test jar: see src/test/resources/testng.xml

    + * + * Used by getTestDataRoot(), projectRoot + * + * @return Path to the project.root directory. + * @throws IllegalStateException if running from a JAR without -Dproject.root defined. + */ + public static Path getProjectRoot() { + // Explicit user override or Maven-injected property + String sysProp = System.getProperty(PROJECT_ROOT_PROP); + if (sysProp != null && !sysProp.isBlank()) { + return Path.of(sysProp).toAbsolutePath().normalize(); + } + + // Strict JAR Execution Rule: Must explicitly provide -Dproject.root = String path + checkExecutionFromJar(PROJECT_ROOT_PROP); + + // Fallback for local source checkout / IDE runs (walk up to top-level .mvn) + Path current = Path.of(System.getProperty("user.dir")).toAbsolutePath(); + while (current != null) { + if (Files.exists(current.resolve(".mvn"))) { + return current.normalize(); + } + current = current.getParent(); + } + throw new IllegalStateException("Could not find project root containing '.mvn' directory"); + } + + private static void checkExecutionFromJar(final String root) { + var url = clazz.getResource(clazz.getSimpleName() + ".class"); + if (url != null && "jar".equals(url.getProtocol())) { + throw new IllegalStateException( + "Executing from a packaged JAR file requires an explicit target directory." + LS + + "Please specify the output directory using: -D" + root + "=/path/to/directory" + ); + } + } + +} diff --git a/src/test/java/org/apache/datasketches/common/UtilityIOTest.java b/src/test/java/org/apache/datasketches/common/UtilityIOTest.java new file mode 100644 index 000000000..78420fb4c --- /dev/null +++ b/src/test/java/org/apache/datasketches/common/UtilityIOTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.common; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.datasketches.common.UtilityIO.getCppPath; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.getJavaPath; +import static org.apache.datasketches.common.UtilityIO.getProjectRoot; +import static org.apache.datasketches.common.UtilityIO.putBytesToFile; +import static org.apache.datasketches.common.UtilityIO.Existence.MUST_EXIST; +import static org.apache.datasketches.common.UtilityIO.Existence.WARNING; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; + +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class UtilityIOTest { + + @Test + public void checkDirCreation() { + assertNotNull(getJavaPath()); + assertNotNull(getCppPath()); + } + + @Test + public void testGetFileBytes_Success() {// throws IOException { + byte[] resultBytes = UtilityIO.getTestResourceBytes("GettysburgAddress.txt"); + assertNotNull(resultBytes); + String resultString = new String(resultBytes, UTF_8); + assertTrue(resultString.startsWith("Abraham Lincoln's Gettysburg Address:")); + } + + @Test + public void testGetFileBytes_MissingFile_Warning() { + byte[] resultBytes = getFileBytes(getProjectRoot(), "Test_NonExistentFile_ThisIsOK", WARNING); + assertNotNull(resultBytes); + assertEquals(resultBytes.length, 0, "Should return empty array for missing file."); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testGetFileBytes_MissingFile_MustExist() { + getFileBytes(getProjectRoot(), "Test_NonExistentFile_ThisIsOK", MUST_EXIST); + } + + @Test(expectedExceptions = RuntimeException.class) + public void testGetFileBytes_NotRegular_NotReadable() throws IOException { + getFileBytes(getProjectRoot(), ""); + } + + private Path tempDir; + + @BeforeMethod + public void setUp() throws IOException { + // Creates a unique temporary directory in the OS temp location + tempDir = Files.createTempDirectory("testDir_"); + } + + @Test + public void testPutBytesToFile() { + byte[] gettysBytes = UtilityIO.getTestResourceBytes("GettysburgAddress.txt"); + putBytesToFile(tempDir, "GettysburgAddressCopy.txt", gettysBytes); + byte[] gettysBytes2 = getFileBytes(tempDir, "GettysburgAddressCopy.txt"); + assertEquals(gettysBytes, gettysBytes2); + } + + @AfterMethod + public void tearDown() throws IOException { + // Recursively deletes the temp directory and all enclosed files after the test runs + if (tempDir != null && Files.exists(tempDir)) { + try (var stream = Files.walk(tempDir)) { + stream.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } + } + +} diff --git a/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java index b0f3ef77f..340ec55ee 100644 --- a/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/cpc/CpcSketchCrossLanguageTest.java @@ -19,18 +19,18 @@ package org.apache.datasketches.cpc; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_GO_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.goPath; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -39,7 +39,7 @@ */ public class CpcSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {0, 100, 200, 2000, 20_000}; final Flavor[] flavorArr = {Flavor.EMPTY, Flavor.SPARSE, Flavor.HYBRID, Flavor.PINNED, Flavor.SLIDING}; @@ -54,7 +54,7 @@ public void generateBinariesForCompatibilityTesting() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) void negativeIntEquivalence() throws Exception { final CpcSketch sk = new CpcSketch(); final byte v1 = (byte) -1; @@ -69,30 +69,46 @@ void negativeIntEquivalence() throws Exception { putBytesToJavaPath("cpc_negative_one_java.sk", sk.toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + allFlavors(GroupLanguage.JAVA); + negativeIntEquivalence(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void allFlavors() throws IOException { - final int[] nArr = {0, 100, 200, 2000, 20000}; - final Flavor[] flavorArr = {Flavor.EMPTY, Flavor.SPARSE, Flavor.HYBRID, Flavor.PINNED, Flavor.SLIDING}; - int flavorIdx = 0; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "cpc_n" + n + "_cpp.sk"); - final CpcSketch sketch = CpcSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getFlavor(), flavorArr[flavorIdx++]); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } + public void checkCpp() { + allFlavors(GroupLanguage.CPP); + negativeIntEquivalence(GroupLanguage.CPP); } @Test(groups = {CHECK_GO_FILES}) - public void checkAllFlavorsGo() throws IOException { + public void checkGo() { + allFlavors(GroupLanguage.GO); + negativeIntEquivalence(GroupLanguage.GO); + } + + private static void allFlavors(final GroupLanguage lang) { final int[] nArr = {0, 100, 200, 2000, 20000}; final Flavor[] flavorArr = {Flavor.EMPTY, Flavor.SPARSE, Flavor.HYBRID, Flavor.PINNED, Flavor.SLIDING}; int flavorIdx = 0; for (final int n: nArr) { - final byte[] bytes = getFileBytes(goPath, "cpc_n" + n + "_go.sk"); + String fileName = "cpc_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); final CpcSketch sketch = CpcSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getFlavor(), flavorArr[flavorIdx++]); assertEquals(sketch.getEstimate(), n, n * 0.02); } } + private static void negativeIntEquivalence(final GroupLanguage lang) { + String fileName = "cpc_negative_one" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final CpcSketch sketch = CpcSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sketch.getEstimate(), 1, 0.02); + } + } diff --git a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java index 4decdb91c..db056aff4 100644 --- a/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/filters/bloomfilter/BloomFilterCrossLanguageTest.java @@ -19,17 +19,18 @@ package org.apache.datasketches.filters.bloomfilter; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; -import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -38,9 +39,9 @@ */ public class BloomFilterCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBloomFilterBinariesForCompatibilityTesting() throws IOException { - final int[] nArr = {0, 10_000, 2_000_000, 300_000_00}; + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateBloomFilterBinaries() { + final int[] nArr = {0, 10_000, 2_000_000, 30_000_000}; final short[] hArr = {3, 5}; for (final int n : nArr) { for (final short numHashes : hArr) { @@ -57,13 +58,30 @@ public void generateBloomFilterBinariesForCompatibilityTesting() throws IOExcept } } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + readBloomFilterBinaries(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void readBloomFilterBinariesForCompatibilityTesting() throws IOException { - final int[] nArr = {0, 10_000, 2_000_000, 300_000_00}; + public void checkCpp() { + readBloomFilterBinaries(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + readBloomFilterBinaries(GroupLanguage.GO); + } + + private static void readBloomFilterBinaries(final GroupLanguage lang) { + final int[] nArr = {0, 10_000, 2_000_000, 30_000_000}; final short[] hArr = {3, 5}; for (final int n : nArr) { for (final short numHashes : hArr) { - final byte[] bytes = getFileBytes(cppPath,"bf_n" + n + "_h" + numHashes + "_cpp.sk"); + final String fileName = "bf_n" + n + "_h" + numHashes + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); final BloomFilter bf = BloomFilter.heapify(MemorySegment.ofArray(bytes)); assertEquals(bf.isEmpty(), n == 0); assertTrue(bf.isEmpty() || (bf.getBitsUsed() > (n / 10))); @@ -77,4 +95,5 @@ public void readBloomFilterBinariesForCompatibilityTesting() throws IOException } } } + } diff --git a/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java index 49a735df7..dfa6b6d44 100644 --- a/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/frequencies/FrequentItemsSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.frequencies; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -32,6 +33,7 @@ import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -40,7 +42,7 @@ */ public class FrequentItemsSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTestingLongsSketch() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -55,7 +57,7 @@ public void generateBinariesForCompatibilityTestingLongsSketch() throws IOExcept } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTestingStringsSketch() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -70,7 +72,7 @@ public void generateBinariesForCompatibilityTestingStringsSketch() throws IOExce } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTestingStringsSketchAscii() throws IOException { final FrequentItemsSketch sk = new FrequentItemsSketch<>(64); sk.update("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1); @@ -80,7 +82,7 @@ public void generateBinariesForCompatibilityTestingStringsSketchAscii() throws I putBytesToJavaPath("frequent_string_ascii_java.sk", sk.toByteArray(new ArrayOfStringsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTestingStringsSketchUtf8() throws IOException { final FrequentItemsSketch sk = new FrequentItemsSketch<>(64); sk.update("абвгд", 1); @@ -93,11 +95,37 @@ public void generateBinariesForCompatibilityTestingStringsSketchUtf8() throws IO putBytesToJavaPath("frequent_string_utf8_java.sk", sk.toByteArray(new ArrayOfStringsSerDe())); } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + longs(GroupLanguage.JAVA); + strings(GroupLanguage.JAVA); + stringsAscii(GroupLanguage.JAVA); + stringsUtf8(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void longs() throws IOException { + public void checkCpp() { + longs(GroupLanguage.CPP); + strings(GroupLanguage.CPP); + stringsAscii(GroupLanguage.JAVA); + stringsUtf8(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + longs(GroupLanguage.GO); + strings(GroupLanguage.GO); + stringsAscii(GroupLanguage.GO); + stringsUtf8(GroupLanguage.GO); + } + + private static void longs(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "frequent_long_n" + n + "_cpp.sk"); + final String fileName = "frequent_long_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final FrequentLongsSketch sketch = FrequentLongsSketch.getInstance(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); if (n > 10) { @@ -109,11 +137,13 @@ public void longs() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void strings() throws IOException { + private static void strings(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "frequent_string_n" + n + "_cpp.sk"); + final String fileName = "frequent_string_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final FrequentItemsSketch sketch = FrequentItemsSketch.getInstance(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); if (n > 10) { @@ -125,9 +155,11 @@ public void strings() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void stringsAscii() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "frequent_string_ascii_cpp.sk"); + private static void stringsAscii(final GroupLanguage lang) { + final String fileName = "frequent_string_ascii" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final FrequentItemsSketch sketch = FrequentItemsSketch.getInstance(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertFalse(sketch.isEmpty()); assertEquals(sketch.getMaximumError(), 0); @@ -138,9 +170,11 @@ public void stringsAscii() throws IOException { assertEquals(sketch.getEstimate("ddddddddddddddddddddddddddddd"), 4); } - @Test(groups = {CHECK_CPP_FILES}) - public void stringsUtf8() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "frequent_string_utf8_cpp.sk"); + private static void stringsUtf8(final GroupLanguage lang) { + final String fileName = "frequent_string_utf8" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final FrequentItemsSketch sketch = FrequentItemsSketch.getInstance(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertFalse(sketch.isEmpty()); assertEquals(sketch.getMaximumError(), 0); diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java index 51932d554..e61966672 100644 --- a/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/hll/HllSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.hll; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.hll.TgtHllType.HLL_4; import static org.apache.datasketches.hll.TgtHllType.HLL_6; import static org.apache.datasketches.hll.TgtHllType.HLL_8; @@ -33,6 +34,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -41,7 +43,7 @@ */ public class HllSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -63,39 +65,35 @@ public void generateBinariesForCompatibilityTesting() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void hll4() throws IOException { - final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "hll4_n" + n + "_cpp.sk"); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeHll(GroupLanguage.JAVA); } @Test(groups = {CHECK_CPP_FILES}) - public void hll6() throws IOException { - final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "hll6_n" + n + "_cpp.sk"); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); - } + public void checkCpp() { + deserializeHll(GroupLanguage.CPP); } - @Test(groups = {CHECK_CPP_FILES}) - public void hll8() throws IOException { + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeHll(GroupLanguage.GO); + } + + private static void deserializeHll(final GroupLanguage lang) { + final String[] sArr = {"hll4", "hll6", "hll8"}; final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; - for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "hll8_n" + n + "_cpp.sk"); - final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); - assertEquals(sketch.getLgConfigK(), 12); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertEquals(sketch.getEstimate(), n, n * 0.02); + for (final String s: sArr) { + for (final int n: nArr) { + final String fileName = s + "_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final HllSketch sketch = HllSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sketch.getLgConfigK(), 12); + assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); + assertEquals(sketch.getEstimate(), n, n * 0.02); + } } } diff --git a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java index 3ce30ec1e..73dfcf8d7 100644 --- a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageTest.java @@ -19,13 +19,13 @@ package org.apache.datasketches.kll; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_HISTORICAL_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; -import static org.apache.datasketches.common.TestUtil.resPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_HISTORICAL_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -35,8 +35,9 @@ import java.util.Comparator; import org.apache.datasketches.common.ArrayOfStringsSerDe; -import org.apache.datasketches.common.TestUtil; import org.apache.datasketches.common.Util; +import org.apache.datasketches.common.UtilityIO; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIteratorAPI; import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIteratorAPI; @@ -49,7 +50,7 @@ public class KllCrossLanguageTest { private final ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe(); - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateKllDoublesSketchBinaries() throws IOException { final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -59,7 +60,7 @@ public void generateKllDoublesSketchBinaries() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateKllFloatsSketchBinaries() throws IOException { final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -69,7 +70,7 @@ public void generateKllFloatsSketchBinaries() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateKllLongsSketchBinaries() throws IOException { final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -79,7 +80,7 @@ public void generateKllLongsSketchBinaries() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateKllItemsSketchBinaries() throws IOException { final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -90,35 +91,38 @@ public void generateKllItemsSketchBinaries() throws IOException { } } - @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) - public void checkCppKllDoublesSketchOneItemVersion1() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "kll_sketch_double_one_item_v1.sk"); - final KllDoublesSketch sk = KllDoublesSketch.heapify(MemorySegment.ofArray(byteArr)); - assertFalse(sk.isEmpty()); - assertFalse(sk.isEstimationMode()); - assertEquals(sk.getN(), 1); - assertEquals(sk.getNumRetained(), 1); - assertEquals(sk.getMinItem(), 1.0); - assertEquals(sk.getMaxItem(), 1.0); + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + kllFloat(GroupLanguage.JAVA); + kllDouble(GroupLanguage.JAVA); + kllString(GroupLanguage.JAVA); + kllLong(GroupLanguage.JAVA); } - @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) - public void checkCppKllFloatsSketchOneItemVersion1() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "kll_sketch_float_one_item_v1.sk"); - final KllFloatsSketch sk = KllFloatsSketch.heapify(MemorySegment.ofArray(byteArr)); - assertFalse(sk.isEmpty()); - assertFalse(sk.isEstimationMode()); - assertEquals(sk.getN(), 1); - assertEquals(sk.getNumRetained(), 1); - assertEquals(sk.getMinItem(), 1.0F); - assertEquals(sk.getMaxItem(), 1.0F); + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + kllFloat(GroupLanguage.CPP); + kllDouble(GroupLanguage.CPP); + kllString(GroupLanguage.CPP); + kllLong(GroupLanguage.CPP); } - @Test(groups = {CHECK_CPP_FILES}) - public void kllFloat() throws IOException { + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + kllFloat(GroupLanguage.GO); + kllDouble(GroupLanguage.GO); + kllString(GroupLanguage.GO); + kllLong(GroupLanguage.GO); + + } + + private static void kllFloat(final GroupLanguage lang) { final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "kll_float_n" + n + "_cpp.sk"); + final String fileName = "kll_float_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllFloatsSketch sketch = KllFloatsSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getK(), 200); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); @@ -139,11 +143,13 @@ public void kllFloat() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void kllDouble() throws IOException { + private static void kllDouble(final GroupLanguage lang) { final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "kll_double_n" + n + "_cpp.sk"); + final String fileName = "kll_double_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllDoublesSketch sketch = KllDoublesSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getK(), 200); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); @@ -164,15 +170,14 @@ public void kllDouble() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void kllString() throws IOException { + private static void kllString(final GroupLanguage lang) { // sketch contains numbers in strings to make meaningful assertions final Comparator numericOrder = new Comparator() { @Override public int compare(final String s1, final String s2) { try { - final int i1 = Integer.parseInt(s1); - final int i2 = Integer.parseInt(s2); + final int i1 = Integer.parseInt(s1.trim()); + final int i2 = Integer.parseInt(s2.trim()); return Integer.compare(i1, i2); } catch (final NumberFormatException e) { throw new RuntimeException(e); @@ -181,7 +186,10 @@ public int compare(final String s1, final String s2) { }; final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "kll_string_n" + n + "_cpp.sk"); + final String fileName = "kll_string_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllHeapItemsSketch sketch = new KllHeapItemsSketch<>( MemorySegment.ofArray(bytes), numericOrder, @@ -192,8 +200,8 @@ public int compare(final String s1, final String s2) { assertTrue(n > 100 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); assertEquals(sketch.getN(), n); if (n > 0) { - assertEquals(sketch.getMinItem(), Integer.toString(1)); - assertEquals(sketch.getMaxItem(), Integer.toString(n)); + assertEquals(sketch.getMinItem().trim(), Integer.toString(1)); + assertEquals(sketch.getMaxItem().trim(), Integer.toString(n)); long weight = 0; final QuantilesGenericSketchIteratorAPI it = sketch.iterator(); while (it.next()) { @@ -206,11 +214,13 @@ public int compare(final String s1, final String s2) { } } - @Test(groups = {CHECK_CPP_FILES}) - public void kllLong() throws IOException { + private static void kllLong(final GroupLanguage lang) { final int[] nArr = {0, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath,"kll_long_n" + n + "_cpp.sk"); + final String fileName = "kll_long_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final KllLongsSketch sketch = KllLongsSketch.heapify(MemorySegment.ofArray(bytes)); assertEquals(sketch.getK(), 200); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); @@ -230,4 +240,31 @@ public void kllLong() throws IOException { } } } + + @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) + public void checkCppKllDoublesSketchOneItemVersion1() { + final String fileName = "kll_sketch_double_one_item_v1.sk"; + final byte[] byteArr = UtilityIO.getTestResourceBytes(fileName); + final KllDoublesSketch sk = KllDoublesSketch.heapify(MemorySegment.ofArray(byteArr)); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertEquals(sk.getMinItem(), 1.0); + assertEquals(sk.getMaxItem(), 1.0); + } + + @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) + public void checkCppKllFloatsSketchOneItemVersion1() { + final String fileName = "kll_sketch_float_one_item_v1.sk"; + final byte[] byteArr = UtilityIO.getTestResourceBytes(fileName); + final KllFloatsSketch sk = KllFloatsSketch.heapify(MemorySegment.ofArray(byteArr)); + assertFalse(sk.isEmpty()); + assertFalse(sk.isEstimationMode()); + assertEquals(sk.getN(), 1); + assertEquals(sk.getNumRetained(), 1); + assertEquals(sk.getMinItem(), 1.0F); + assertEquals(sk.getMaxItem(), 1.0F); + } + } diff --git a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java index 53746c260..1c03fb03b 100644 --- a/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/quantiles/QuantilesSketchCrossLanguageTest.java @@ -19,13 +19,13 @@ package org.apache.datasketches.quantiles; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_HISTORICAL_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; -import static org.apache.datasketches.common.TestUtil.resPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_HISTORICAL_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.EXCLUSIVE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -35,7 +35,8 @@ import java.util.Comparator; import org.apache.datasketches.common.ArrayOfStringsSerDe; -import org.apache.datasketches.common.TestUtil; +import org.apache.datasketches.common.UtilityIO; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.quantilescommon.QuantilesDoublesSketchIteratorAPI; import org.apache.datasketches.quantilescommon.QuantilesGenericSketchIteratorAPI; import org.testng.Assert; @@ -48,7 +49,7 @@ public class QuantilesSketchCrossLanguageTest { private static final String LS = System.getProperty("line.separator"); - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateDoublesSketch() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -56,11 +57,11 @@ public void generateDoublesSketch() throws IOException { for (int i = 1; i <= n; i++) { sk.update(i); } - putBytesToJavaPath("quantiles_double_n" + n + "_java.sk", sk.toByteArray()); + putBytesToJavaPath("quantiles_double_n" + n + "_java.sk", sk.toByteArray(true)); } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateItemsSketchWithStrings() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -87,12 +88,32 @@ public int compare(final String s1, final String s2) { } } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + checkDoublesSketch(GroupLanguage.JAVA); + checkItemsSketchWithStrings(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void checkDoublesSketch() throws IOException { + public void checkCpp() { + checkDoublesSketch(GroupLanguage.CPP); + checkItemsSketchWithStrings(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + checkDoublesSketch(GroupLanguage.GO); + checkItemsSketchWithStrings(GroupLanguage.GO); + } + + private static void checkDoublesSketch(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] byteArr = getFileBytes(cppPath, "quantiles_double_n" + n + "_cpp.sk"); - final QuantilesDoublesSketch sk = QuantilesDoublesSketch.wrap(MemorySegment.ofArray(byteArr)); + final String fileName = "quantiles_double_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final QuantilesDoublesSketch sk = QuantilesDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); assertTrue(n > 128 ? sk.isEstimationMode() : !sk.isEstimationMode()); assertEquals(sk.getN(), n); @@ -111,8 +132,7 @@ public void checkDoublesSketch() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void checkItemsSketchWithStrings() throws IOException { + private static void checkItemsSketchWithStrings(final GroupLanguage lang) { // sketch contains numbers in strings to make meaningful assertions final Comparator numericOrder = new Comparator() { @Override @@ -128,10 +148,13 @@ public int compare(final String s1, final String s2) { }; final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] byteArr = getFileBytes(cppPath, "quantiles_string_n" + n + "_cpp.sk"); + final String fileName = "quantiles_string_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final QuantilesItemsSketch sk = QuantilesItemsSketch.heapify( String.class, - MemorySegment.ofArray(byteArr), + MemorySegment.ofArray(bytes), numericOrder, new ArrayOfStringsSerDe() ); @@ -242,7 +265,7 @@ private static void getAndCheck(final String ver, final int n, final double quan println("fullName: "+ fileName); println("Old Median: " + quantile); //Read File bytes - final byte[] byteArr = TestUtil.getFileBytes(resPath, fileName); + final byte[] byteArr = UtilityIO.getTestResourceBytes(fileName); final MemorySegment srcSeg = MemorySegment.ofArray(byteArr); // heapify as update sketch diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java index 8e38b9b16..cc2f6633f 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchCrossLanguageTest.java @@ -19,17 +19,19 @@ package org.apache.datasketches.req; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.quantilescommon.QuantilesFloatsSketchIterator; import org.testng.annotations.Test; @@ -39,7 +41,7 @@ */ public class ReqSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -51,7 +53,7 @@ public void generateBinariesForCompatibilityTesting() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateNegativeBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {1, 10}; for (final int n: nArr) { @@ -63,23 +65,46 @@ public void generateNegativeBinariesForCompatibilityTesting() throws IOException } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateMixedBinariesForCompatibilityTesting() throws IOException { final int[] nArr = {1, 10}; for (final int n: nArr) { final ReqSketch sk = ReqSketch.builder().build(); - for (int i = -n; i <= n; i++) { + for (int i = -n; i <= n; i++) { //produces sets of {-1,0,1}, {-10...,0,...10} sk.update(i); } putBytesToJavaPath("req_float_mixed_n" + n + "_java.sk", sk.toByteArray()); } } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + checkReqPositiveValues(GroupLanguage.JAVA); + checkReqNegativeValues(GroupLanguage.JAVA); + checkReqMixedValues(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCpp() throws IOException { + public void checkCpp() { + checkReqPositiveValues(GroupLanguage.CPP); + checkReqNegativeValues(GroupLanguage.CPP); + checkReqMixedValues(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + checkReqPositiveValues(GroupLanguage.GO); + checkReqNegativeValues(GroupLanguage.GO); + checkReqMixedValues(GroupLanguage.GO); + } + + private static void checkReqPositiveValues(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "req_float_n" + n + "_cpp.sk"); + final String fileName = "req_float_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); assertTrue(n > 10 ? sk.isEstimationMode() : !sk.isEstimationMode()); @@ -98,4 +123,56 @@ public void deserializeFromCpp() throws IOException { } } } + + private static void checkReqNegativeValues(final GroupLanguage lang) { + final int[] nArr = {1, 10}; + for (final int n: nArr) { + final String fileName = "req_float_negative_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); + assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); + assertTrue(n > 10 ? sk.isEstimationMode() : !sk.isEstimationMode()); + assertEquals(sk.getN(), n); + if (n > 0) { + assertEquals(sk.getMinItem(), -n); + assertEquals(sk.getMaxItem(), -1); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while(it.next()) { + assertTrue(it.getQuantile() >= sk.getMinItem()); + assertTrue(it.getQuantile() <= sk.getMaxItem()); + weight += it.getWeight(); + } + assertEquals(weight, n); + } + } + } + + private static void checkReqMixedValues(final GroupLanguage lang) { + final int[] nArr = {1, 10}; + for (final int n: nArr) { + final String fileName = "req_float_mixed_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReqSketch sk = ReqSketch.heapify(MemorySegment.ofArray(bytes)); + assertTrue(n == 0 ? sk.isEmpty() : !sk.isEmpty()); + if (n == 1) { assertEquals(sk.getN(), 3); } + if (n == 10) { assertEquals(sk.getN(), 21); } + if (n > 0) { + assertEquals(sk.getMinItem(), -n); + assertEquals(sk.getMaxItem(), n); + final QuantilesFloatsSketchIterator it = sk.iterator(); + long weight = 0; + while(it.next()) { + assertTrue(it.getQuantile() >= sk.getMinItem()); + assertTrue(it.getQuantile() <= sk.getMaxItem()); + weight += it.getWeight(); + } + assertEquals(weight, 2L * n + 1L); + } + } + } } diff --git a/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java b/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java index 564f3b53b..37b1d91d4 100644 --- a/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/sampling/ReservoirCrossLanguageTest.java @@ -19,16 +19,24 @@ package org.apache.datasketches.sampling; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; import java.io.IOException; +import java.lang.foreign.MemorySegment; import java.util.ArrayList; import org.apache.datasketches.common.ArrayOfDoublesSerDe; import org.apache.datasketches.common.ArrayOfLongsSerDe; import org.apache.datasketches.common.ArrayOfStringsSerDe; import org.apache.datasketches.common.ResizeFactor; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -37,16 +45,15 @@ */ public class ReservoirCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsSketchEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirLongsSketchEmpty() throws IOException { //1 final int k = 128; final ReservoirLongsSketch sk = ReservoirLongsSketch.newInstance(k); - putBytesToJavaPath("reservoir_longs_empty_k" + k + "_java.sk", sk.toByteArray()); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsSketchExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirLongsSketchExact() throws IOException { //2 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -55,12 +62,13 @@ public void generateReservoirLongsSketchExact() throws IOException { for (int i = 0; i < n; i++) { sk.update(i); } + putBytesToJavaPath("reservoir_longs_exact_n" + n + "_k" + k + "_java.sk", sk.toByteArray()); } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsSketchSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirLongsSketchSampling() throws IOException { //3 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -81,16 +89,16 @@ public void generateReservoirLongsSketchSampling() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsUnionEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirLongsUnionEmpty() throws IOException { //4 int maxK = 128; ReservoirLongsUnion union = ReservoirLongsUnion.newInstance(maxK); putBytesToJavaPath("reservoir_longs_union_empty_maxk" + maxK + "_java.sk", union.toByteArray()); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsUnionExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirLongsUnionExact() throws IOException { //5 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -103,8 +111,8 @@ public void generateReservoirLongsUnionExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirLongsUnionSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirLongsUnionSampling() throws IOException { //6 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -128,16 +136,16 @@ public void generateReservoirLongsUnionSampling() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchLongEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchLongEmpty() throws IOException { //7 final int k = 128; final ReservoirItemsSketch sk = ReservoirItemsSketch.newInstance(k); putBytesToJavaPath("reservoir_items_long_empty_k" + k + "_java.sk", sk.toByteArray(new ArrayOfLongsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchLongExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchLongExact() throws IOException { //8 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -150,8 +158,8 @@ public void generateReservoirItemsSketchLongExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchLongSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchLongSampling() throws IOException { //9 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -172,16 +180,16 @@ public void generateReservoirItemsSketchLongSampling() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchDoubleEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchDoubleEmpty() throws IOException { //10 final int k = 128; final ReservoirItemsSketch sk = ReservoirItemsSketch.newInstance(k); putBytesToJavaPath("reservoir_items_double_empty_k" + k + "_java.sk", sk.toByteArray(new ArrayOfDoublesSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchDoubleExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchDoubleExact() throws IOException { //11 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -194,8 +202,8 @@ public void generateReservoirItemsSketchDoubleExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchDoubleSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchDoubleSampling() throws IOException { //12 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -216,16 +224,16 @@ public void generateReservoirItemsSketchDoubleSampling() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchStringEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchStringEmpty() throws IOException { //13 final int k = 128; final ReservoirItemsSketch sk = ReservoirItemsSketch.newInstance(k); putBytesToJavaPath("reservoir_items_string_empty_k" + k + "_java.sk", sk.toByteArray(new ArrayOfStringsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchStringExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchStringExact() throws IOException { //14 final int k = 128; final int[] nArr = {1, 10, 32, 100, 128}; @@ -238,8 +246,8 @@ public void generateReservoirItemsSketchStringExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsSketchStringSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsSketchStringSampling() throws IOException { //15 final int[] kArr = {32, 64, 128}; final long n = 1000; @@ -255,21 +263,20 @@ public void generateReservoirItemsSketchStringSampling() throws IOException { ResizeFactor.X8, k ); - putBytesToJavaPath("reservoir_items_string_sampling_n" + n + "_k" + k + "_java.sk", sk.toByteArray(new ArrayOfStringsSerDe())); } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionLongEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionLongEmpty() throws IOException { //16 int maxK = 128; ReservoirItemsUnion union = ReservoirItemsUnion.newInstance(maxK); putBytesToJavaPath("reservoir_items_union_long_empty_maxk" + maxK + "_java.sk", union.toByteArray(new ArrayOfLongsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionLongExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionLongExact() throws IOException { //17 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -283,8 +290,8 @@ public void generateReservoirItemsUnionLongExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionLongSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionLongSampling() throws IOException { //18 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -309,16 +316,16 @@ public void generateReservoirItemsUnionLongSampling() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionDoubleEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionDoubleEmpty() throws IOException { //19 int maxK = 128; ReservoirItemsUnion union = ReservoirItemsUnion.newInstance(maxK); putBytesToJavaPath("reservoir_items_union_double_empty_maxk" + maxK + "_java.sk", union.toByteArray(new ArrayOfDoublesSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionDoubleExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionDoubleExact() throws IOException { //20 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -332,8 +339,8 @@ public void generateReservoirItemsUnionDoubleExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionDoubleSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionDoubleSampling() throws IOException { //21 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -358,16 +365,16 @@ public void generateReservoirItemsUnionDoubleSampling() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionStringEmpty() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionStringEmpty() throws IOException { //22 int maxK = 128; ReservoirItemsUnion union = ReservoirItemsUnion.newInstance(maxK); putBytesToJavaPath("reservoir_items_union_string_empty_maxk" + maxK + "_java.sk", union.toByteArray(new ArrayOfStringsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionStringExact() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionStringExact() throws IOException { //23 int maxK = 128; int[] nArr = {1, 10, 32, 100, 128}; @@ -381,8 +388,8 @@ public void generateReservoirItemsUnionStringExact() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateReservoirItemsUnionStringSampling() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateReservoirItemsUnionStringSampling() throws IOException { //24 int[] maxKArr = {32, 64, 128}; long n = 1000; @@ -406,4 +413,464 @@ public void generateReservoirItemsUnionStringSampling() throws IOException { union.toByteArray(new ArrayOfStringsSerDe())); } } + /*****************************************************/ + /*****************************************************/ + + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + checkReservoirLongsSketchEmpty(GroupLanguage.JAVA); + checkReservoirLongsSketchExact(GroupLanguage.JAVA); + checkReservoirLongsSketchSampling(GroupLanguage.JAVA); + checkReservoirLongsUnionEmpty(GroupLanguage.JAVA); + checkReservoirLongsUnionExact(GroupLanguage.JAVA); + checkReservoirLongsUnionSampling(GroupLanguage.JAVA); + checkReservoirItemsSketchLongEmpty(GroupLanguage.JAVA); + checkReservoirItemsSketchLongExact(GroupLanguage.JAVA); + checkReservoirItemsSketchLongSampling(GroupLanguage.JAVA); + checkReservoirItemsSketchDoubleEmpty(GroupLanguage.JAVA); + checkReservoirItemsSketchDoubleExact(GroupLanguage.JAVA); + checkReservoirItemsSketchDoubleSampling(GroupLanguage.JAVA); + checkReservoirItemsSketchStringEmpty(GroupLanguage.JAVA); + checkReservoirItemsSketchStringExact(GroupLanguage.JAVA); + checkReservoirItemsSketchStringSampling(GroupLanguage.JAVA); + checkReservoirItemsUnionLongEmpty(GroupLanguage.JAVA); + checkReservoirItemsUnionLongExact(GroupLanguage.JAVA); + checkReservoirItemsUnionLongSampling(GroupLanguage.JAVA); + checkReservoirItemsUnionDoubleEmpty(GroupLanguage.JAVA); + checkReservoirItemsUnionDoubleExact(GroupLanguage.JAVA); + checkReservoirItemsUnionDoubleSampling(GroupLanguage.JAVA); + checkReservoirItemsUnionStringEmpty(GroupLanguage.JAVA); + checkReservoirItemsUnionStringExact(GroupLanguage.JAVA); + checkReservoirItemsUnionStringSampling(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + checkReservoirLongsSketchEmpty(GroupLanguage.CPP); + checkReservoirLongsSketchExact(GroupLanguage.CPP); + checkReservoirLongsSketchSampling(GroupLanguage.CPP); + checkReservoirLongsUnionEmpty(GroupLanguage.CPP); + checkReservoirLongsUnionExact(GroupLanguage.CPP); + checkReservoirLongsUnionSampling(GroupLanguage.CPP); + checkReservoirItemsSketchLongEmpty(GroupLanguage.CPP); + checkReservoirItemsSketchLongExact(GroupLanguage.CPP); + checkReservoirItemsSketchLongSampling(GroupLanguage.CPP); + checkReservoirItemsSketchDoubleEmpty(GroupLanguage.CPP); + checkReservoirItemsSketchDoubleExact(GroupLanguage.CPP); + checkReservoirItemsSketchDoubleSampling(GroupLanguage.CPP); + checkReservoirItemsSketchStringEmpty(GroupLanguage.CPP); + checkReservoirItemsSketchStringExact(GroupLanguage.CPP); + checkReservoirItemsSketchStringSampling(GroupLanguage.CPP); + checkReservoirItemsUnionLongEmpty(GroupLanguage.CPP); + checkReservoirItemsUnionLongExact(GroupLanguage.CPP); + checkReservoirItemsUnionLongSampling(GroupLanguage.CPP); + checkReservoirItemsUnionDoubleEmpty(GroupLanguage.CPP); + checkReservoirItemsUnionDoubleExact(GroupLanguage.CPP); + checkReservoirItemsUnionDoubleSampling(GroupLanguage.CPP); + checkReservoirItemsUnionStringEmpty(GroupLanguage.CPP); + checkReservoirItemsUnionStringExact(GroupLanguage.CPP); + checkReservoirItemsUnionStringSampling(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + checkReservoirLongsSketchEmpty(GroupLanguage.GO); + checkReservoirLongsSketchExact(GroupLanguage.GO); + checkReservoirLongsSketchSampling(GroupLanguage.GO); + checkReservoirLongsUnionEmpty(GroupLanguage.GO); + checkReservoirLongsUnionExact(GroupLanguage.GO); + checkReservoirLongsUnionSampling(GroupLanguage.GO); + checkReservoirItemsSketchLongEmpty(GroupLanguage.GO); + checkReservoirItemsSketchLongExact(GroupLanguage.GO); + checkReservoirItemsSketchLongSampling(GroupLanguage.GO); + checkReservoirItemsSketchDoubleEmpty(GroupLanguage.GO); + checkReservoirItemsSketchDoubleExact(GroupLanguage.GO); + checkReservoirItemsSketchDoubleSampling(GroupLanguage.GO); + checkReservoirItemsSketchStringEmpty(GroupLanguage.GO); + checkReservoirItemsSketchStringExact(GroupLanguage.GO); + checkReservoirItemsSketchStringSampling(GroupLanguage.GO); + checkReservoirItemsUnionLongEmpty(GroupLanguage.GO); + checkReservoirItemsUnionLongExact(GroupLanguage.GO); + checkReservoirItemsUnionLongSampling(GroupLanguage.GO); + checkReservoirItemsUnionDoubleEmpty(GroupLanguage.GO); + checkReservoirItemsUnionDoubleExact(GroupLanguage.GO); + checkReservoirItemsUnionDoubleSampling(GroupLanguage.GO); + checkReservoirItemsUnionStringEmpty(GroupLanguage.GO); + checkReservoirItemsUnionStringExact(GroupLanguage.GO); + checkReservoirItemsUnionStringSampling(GroupLanguage.GO); + } + + //ReservoirLongsSketch + + private static void checkReservoirLongsSketchEmpty(final GroupLanguage lang) { //1 + final int k = 128; + final String fileName = "reservoir_longs_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirLongsSketch sk = ReservoirLongsSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirLongsSketchExact(final GroupLanguage lang) { //1 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (final int n : nArr) { + final String fileName = "reservoir_longs_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsSketch sk = ReservoirLongsSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirLongsSketchSampling(final GroupLanguage lang) { //3 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_longs_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsSketch sk = ReservoirLongsSketch.heapify(MemorySegment.ofArray(bytes)); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirLongsUnion + + private static void checkReservoirLongsUnionEmpty(final GroupLanguage lang) { //4 + int maxK = 128; + final String fileName = "reservoir_longs_union_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirLongsUnion rlu = ReservoirLongsUnion.heapify(MemorySegment.ofArray(bytes)); + assertEquals(rlu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirLongsUnionExact(final GroupLanguage lang) { //5 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_longs_union_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsUnion rlu = ReservoirLongsUnion.heapify(MemorySegment.ofArray(bytes)); + assertEquals(rlu.getMaxK(), maxK); + final ReservoirLongsSketch sk = rlu.getResult(); + assertTrue(sk.getN() == n); + assertTrue(sk.getNumSamples() == n); + } + } + + private static void checkReservoirLongsUnionSampling(final GroupLanguage lang) { //6 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_longs_union_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirLongsUnion rlu = ReservoirLongsUnion.heapify(MemorySegment.ofArray(bytes)); + assertEquals(rlu.getMaxK(), maxK); + final ReservoirLongsSketch sk = rlu.getResult(); + assertTrue(sk.getN() == n); + assertTrue(sk.getNumSamples() == maxK); + } + } + + //ReservoirItemsSketch + + private static void checkReservoirItemsSketchLongEmpty(final GroupLanguage lang) { //7 + final int k = 128; + final String fileName = "reservoir_items_long_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirItemsSketchLongExact(final GroupLanguage lang) { //8 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_long_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsSketchLongSampling(final GroupLanguage lang) { //9 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_items_long_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirItemsSketch + + private static void checkReservoirItemsSketchDoubleEmpty(final GroupLanguage lang) { //10 + final int k = 128; + final String fileName = "reservoir_items_double_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirItemsSketchDoubleExact(final GroupLanguage lang) { //11 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_double_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsSketchDoubleSampling(final GroupLanguage lang) { //12 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_items_double_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirItemsSketch + + private static void checkReservoirItemsSketchStringEmpty(final GroupLanguage lang) { //13 + final int k = 128; + final String fileName = "reservoir_items_string_empty_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), 0); + assertEquals(sk.getNumSamples(), 0); + } + + private static void checkReservoirItemsSketchStringExact(final GroupLanguage lang) { //14 + final int k = 128; + final int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_string_exact_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsSketchStringSampling(final GroupLanguage lang) { //15 + final int[] kArr = {32, 64, 128}; + final long n = 1000; + for (final int k : kArr) { + final String fileName = "reservoir_items_string_sampling_n" + n + "_k" + k + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsSketch sk = ReservoirItemsSketch.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(sk.getK(), k); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), k); + } + } + + //ReservoirItemsUnion + + private static void checkReservoirItemsUnionLongEmpty(final GroupLanguage lang) { //16 + int maxK = 128; + final String fileName = "reservoir_items_union_long_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(riu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirItemsUnionLongExact(final GroupLanguage lang) { //17 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_union_long_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsUnionLongSampling(final GroupLanguage lang) { //18 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_items_union_long_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), maxK); + } + } + + //ReservoirItemsUnion + + private static void checkReservoirItemsUnionDoubleEmpty(final GroupLanguage lang) { //19 + int maxK = 128; + final String fileName = "reservoir_items_union_double_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(riu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirItemsUnionDoubleExact(final GroupLanguage lang) { //20 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_union_double_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsUnionDoubleSampling(final GroupLanguage lang) { //21 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_items_union_double_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), maxK); + } + } + + //ReservoirItemsUnion + + private static void checkReservoirItemsUnionStringEmpty(final GroupLanguage lang) { //22 + int maxK = 128; + final String fileName = "reservoir_items_union_string_empty_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(riu.getMaxK(), maxK); + //gadget is null + } + + private static void checkReservoirItemsUnionStringExact(final GroupLanguage lang) { //23 + int maxK = 128; + int[] nArr = {1, 10, 32, 100, 128}; + for (int n : nArr) { + final String fileName = "reservoir_items_union_string_exact_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), n); + } + } + + private static void checkReservoirItemsUnionStringSampling(final GroupLanguage lang) { //24 + int[] maxKArr = {32, 64, 128}; + long n = 1000; + for (int maxK : maxKArr) { + final String fileName = "reservoir_items_union_string_sampling_n" + n + "_maxk" + maxK + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final ReservoirItemsUnion riu = ReservoirItemsUnion.heapify( + MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); + assertEquals(riu.getMaxK(), maxK); + final ReservoirItemsSketch sk = riu.getResult(); + assertEquals(sk.getN(), n); + assertEquals(sk.getNumSamples(), maxK); + } + } } diff --git a/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java b/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java index 169245d4b..06bece134 100644 --- a/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/sampling/VarOptCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.sampling; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -33,6 +34,7 @@ import org.apache.datasketches.common.ArrayOfDoublesSerDe; import org.apache.datasketches.common.ArrayOfLongsSerDe; import org.apache.datasketches.common.ArrayOfStringsSerDe; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -42,7 +44,7 @@ public class VarOptCrossLanguageTest { static final double EPS = 1e-13; - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateSketchesLong() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { @@ -54,7 +56,7 @@ public void generateSketchesLong() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateSketchStringExact() throws IOException { final VarOptItemsSketch sketch = VarOptItemsSketch.newInstance(1024); for (int i = 1; i <= 200; ++i) { @@ -63,7 +65,7 @@ public void generateSketchStringExact() throws IOException { putBytesToJavaPath("varopt_sketch_string_exact_java.sk", sketch.toByteArray(new ArrayOfStringsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateSketchLongSampling() throws IOException { final VarOptItemsSketch sketch = VarOptItemsSketch.newInstance(1024); for (long i = 0; i < 2000; ++i) { @@ -76,7 +78,7 @@ public void generateSketchLongSampling() throws IOException { putBytesToJavaPath("varopt_sketch_long_sampling_java.sk", sketch.toByteArray(new ArrayOfLongsSerDe())); } - @Test(groups = {GENERATE_JAVA_FILES}) + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) public void generateUnionDoubleSampling() throws IOException { final int kSmall = 16; final int n1 = 32; @@ -103,11 +105,37 @@ public void generateUnionDoubleSampling() throws IOException { putBytesToJavaPath("varopt_union_double_sampling_java.sk", union.toByteArray(new ArrayOfDoublesSerDe())); } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeSketchLongs(GroupLanguage.JAVA); + deserializeSketchStringsExact(GroupLanguage.JAVA); + deserializeSketdhLongsSampling(GroupLanguage.JAVA); + deserializeUnionDoubleSampling(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppSketchLongs() throws IOException { + public void checkCpp() { + deserializeSketchLongs(GroupLanguage.CPP); + deserializeSketchStringsExact(GroupLanguage.CPP); + deserializeSketdhLongsSampling(GroupLanguage.CPP); + deserializeUnionDoubleSampling(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeSketchLongs(GroupLanguage.GO); + deserializeSketchStringsExact(GroupLanguage.GO); + deserializeSketdhLongsSampling(GroupLanguage.GO); + deserializeUnionDoubleSampling(GroupLanguage.GO); + } + + private static void deserializeSketchLongs(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "varopt_sketch_long_n" + n + "_cpp.sk"); + final String fileName = "varopt_sketch_long_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final VarOptItemsSketch sk = VarOptItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); assertEquals(sk.getK(), 32); assertEquals(sk.getN(), n); @@ -115,10 +143,12 @@ public void deserializeFromCppSketchLongs() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) @SuppressWarnings("unused") - public void deserializeFromCppSketchStringsExact() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "varopt_sketch_string_exact_cpp.sk"); + private static void deserializeSketchStringsExact(final GroupLanguage lang) { + final String fileName = "varopt_sketch_string_exact" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final VarOptItemsSketch sk = VarOptItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfStringsSerDe()); assertEquals(sk.getK(), 1024); assertEquals(sk.getN(), 200); @@ -131,10 +161,12 @@ public void deserializeFromCppSketchStringsExact() throws IOException { assertEquals(ss.getTotalSketchWeight(), weight, EPS); } - @Test(groups = {CHECK_CPP_FILES}) @SuppressWarnings("unused") - public void deserializeFromCppSketchLongsSampling() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "varopt_sketch_long_sampling_cpp.sk"); + private static void deserializeSketdhLongsSampling(final GroupLanguage lang) { + String fileName = "varopt_sketch_long_sampling" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final VarOptItemsSketch sk = VarOptItemsSketch.heapify(MemorySegment.ofArray(bytes), new ArrayOfLongsSerDe()); assertEquals(sk.getK(), 1024); assertEquals(sk.getN(), 2003); @@ -149,9 +181,11 @@ public void deserializeFromCppSketchLongsSampling() throws IOException { assertEquals(ss.getEstimate(), 2000.0, EPS); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppUnionDoubleSampling() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "varopt_union_double_sampling_cpp.sk"); + private static void deserializeUnionDoubleSampling(final GroupLanguage lang) { + final String fileName = "varopt_union_double_sampling" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final VarOptItemsUnion u = VarOptItemsUnion.heapify(MemorySegment.ofArray(bytes), new ArrayOfDoublesSerDe()); // must reduce k in the process diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java index 99d88b429..30327e434 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestCrossLanguageTest.java @@ -19,90 +19,77 @@ package org.apache.datasketches.tdigest; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; public class TDigestCrossLanguageTest { - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppDouble() throws IOException { - final boolean[] with_buffer = {false, true}; - for (final boolean buffered : with_buffer) { - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (final int n : nArr) { - final byte[] bytes; - if (buffered) { - bytes = getFileBytes(cppPath, "tdigest_double_buf_n" + n + "_cpp.sk"); - } else { - bytes = getFileBytes(cppPath, "tdigest_double_n" + n + "_cpp.sk"); - } - final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); - assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); - assertEquals(td.getTotalWeight(), n); - if (n > 0) { - assertEquals(td.getMinValue(), 1); - assertEquals(td.getMaxValue(), n); - assertEquals(td.getRank(0), 0); - assertEquals(td.getRank(n + 1), 1); - if (n == 1) { - assertEquals(td.getRank(n), 0.5); - } else { - assertEquals(td.getRank(n / 2), 0.5, 0.05); - } - } + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateForCppDouble() throws IOException { + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (final int n : nArr) { + final TDigestDouble td = new TDigestDouble((short) 100); + for (int i = 1; i <= n; i++) { + td.update(i); } + putBytesToJavaPath("tdigest_double_n" + n + "_java.sk", td.toByteArray()); } } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeTDigest(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppFloat() throws IOException { - final boolean[] with_buffer = {false, true}; - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (final boolean buffered : with_buffer) { - for (final int n : nArr) { - final byte[] bytes; - if (buffered) { - bytes = getFileBytes(cppPath, "tdigest_float_buf_n" + n + "_cpp.sk"); - } else { - bytes = getFileBytes(cppPath, "tdigest_float_n" + n + "_cpp.sk"); - } - final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), true); - assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); - assertEquals(td.getTotalWeight(), n); - if (n > 0) { - assertEquals(td.getMinValue(), 1); - assertEquals(td.getMaxValue(), n); - assertEquals(td.getRank(0), 0); - assertEquals(td.getRank(n + 1), 1); - if (n == 1) { - assertEquals(td.getRank(n), 0.5); - } else { - assertEquals(td.getRank(n / 2), 0.5, 0.05); - } - } - } - } + public void checkCpp() { + deserializeTDigest(GroupLanguage.CPP); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateForCppDouble() throws IOException { + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeTDigest(GroupLanguage.GO); + } + + private static void deserializeTDigest(final GroupLanguage lang) { + final String[] dfArr = {"double_", "float_"}; + final String[] bufArr = {"buf_", ""}; final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (final int n : nArr) { - final TDigestDouble td = new TDigestDouble((short) 100); - for (int i = 1; i <= n; i++) { - td.update(i); + for (final String df: dfArr) { + for (final String buf: bufArr) { + for (final int n : nArr) { + final String fileName = "tdigest_" + df + buf + "n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes), df == "float_"); + assertTrue(n == 0 ? td.isEmpty() : !td.isEmpty()); + assertEquals(td.getTotalWeight(), n); + if (n > 0) { + assertEquals(td.getMinValue(), 1); + assertEquals(td.getMaxValue(), n); + assertEquals(td.getRank(0), 0); + assertEquals(td.getRank(n + 1), 1); + if (n == 1) { + assertEquals(td.getRank(n), 0.5); + } else { + assertEquals(td.getRank(n / 2), 0.5, 0.05); + } + } + } } - putBytesToJavaPath("tdigest_double_n" + n + "_java.sk", td.toByteArray()); } } diff --git a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java index 18e4103cf..5b565074b 100644 --- a/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java +++ b/src/test/java/org/apache/datasketches/tdigest/TDigestDoubleTest.java @@ -19,7 +19,6 @@ package org.apache.datasketches.tdigest; -import static org.apache.datasketches.common.TestUtil.resPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertThrows; @@ -30,7 +29,7 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.common.SketchesStateException; -import org.apache.datasketches.common.TestUtil; +import org.apache.datasketches.common.UtilityIO; import org.testng.annotations.Test; public class TDigestDoubleTest { @@ -238,7 +237,7 @@ public void deserializeNaNSingleValue() { @Test public void deserializeFromReferenceImplementationDouble() { - final byte[] bytes = TestUtil.getFileBytes(resPath, "tdigest_ref_k100_n10000_double.sk"); + final byte[] bytes = UtilityIO.getTestResourceBytes("tdigest_ref_k100_n10000_double.sk"); final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); final int n = 10000; assertEquals(td.getK(), 100); @@ -254,7 +253,7 @@ public void deserializeFromReferenceImplementationDouble() { @Test public void deserializeFromReferenceImplementationFloat() { - final byte[] bytes = TestUtil.getFileBytes(resPath, "tdigest_ref_k100_n10000_float.sk"); + final byte[] bytes = UtilityIO.getTestResourceBytes("tdigest_ref_k100_n10000_float.sk"); final TDigestDouble td = TDigestDouble.heapify(MemorySegment.ofArray(bytes)); final int n = 10000; assertEquals(td.getK(), 100); diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java index c9cf0361b..5bf39f7ee 100644 --- a/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.theta; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -31,6 +32,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -39,8 +41,8 @@ */ public class ThetaSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTesting() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeSketches() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { final UpdatableThetaSketch sk = UpdatableThetaSketch.builder().build(); @@ -51,8 +53,8 @@ public void generateBinariesForCompatibilityTesting() throws IOException { } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingCompressed() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeCompressedSketches() throws IOException { final int[] nArr = {10, 100, 1000, 10_000, 100_000, 1_000_000}; for (final int n: nArr) { final UpdatableThetaSketch sk = UpdatableThetaSketch.builder().build(); @@ -63,20 +65,52 @@ public void generateBinariesForCompatibilityTestingCompressed() throws IOExcepti } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeNonEmptyNoEntries() throws IOException { final UpdatableThetaSketch sk = UpdatableThetaSketch.builder().setP(0.01f).build(); - sk.update(1); + sk.update(1); //ignored because of p = .01. assertFalse(sk.isEmpty()); assertEquals(sk.getRetainedEntries(), 0); putBytesToJavaPath("theta_non_empty_no_entries_java.sk", sk.compact().toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeSketchesUsingSegment(GroupLanguage.JAVA); + deserializeSketchesFromFile(GroupLanguage.JAVA); + deserializeCompressedUsingSegment(GroupLanguage.JAVA); + deserializeCompressedFromFile(GroupLanguage.JAVA); + deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.JAVA); + deserializeNonEmptyNoEntriesFromFile(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppSegment() throws IOException { + public void checkCpp() { + deserializeSketchesUsingSegment(GroupLanguage.CPP); + deserializeSketchesFromFile(GroupLanguage.CPP); + deserializeCompressedUsingSegment(GroupLanguage.CPP); + deserializeCompressedFromFile(GroupLanguage.CPP); + deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.CPP); + deserializeNonEmptyNoEntriesFromFile(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeSketchesUsingSegment(GroupLanguage.GO); + deserializeSketchesFromFile(GroupLanguage.GO); + deserializeCompressedUsingSegment(GroupLanguage.GO); + deserializeCompressedFromFile(GroupLanguage.GO); + deserializeNonEmptyNoEntriesUsingSegment(GroupLanguage.GO); + deserializeNonEmptyNoEntriesFromFile(GroupLanguage.GO); + } + + private static void deserializeSketchesUsingSegment(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_n" + n + "_cpp.sk"); + final String fileName = "theta_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -91,11 +125,13 @@ public void deserializeFromCppSegment() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppBytes() throws IOException { + private static void deserializeSketchesFromFile(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_n" + n + "_cpp.sk"); + final String fileName = "theta_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(bytes); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -110,11 +146,13 @@ public void deserializeFromCppBytes() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppCompressedSegment() throws IOException { + private static void deserializeCompressedUsingSegment(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_compressed_n" + n + "_cpp.sk"); + final String fileName = "theta_compressed_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -129,11 +167,13 @@ public void deserializeFromCppCompressedSegment() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppCompressedBytes() throws IOException { + private static void deserializeCompressedFromFile(final GroupLanguage lang) { final int[] nArr = {10, 100, 1000, 10000, 100000, 1000000}; for (final int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "theta_compressed_n" + n + "_cpp.sk"); + final String fileName = "theta_compressed_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(bytes); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -148,17 +188,21 @@ public void deserializeFromCppCompressedBytes() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppNonEmptyNoEntriesSegment() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "theta_non_empty_no_entries_cpp.sk"); + private static void deserializeNonEmptyNoEntriesUsingSegment(final GroupLanguage lang) { + final String fileName = "theta_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(MemorySegment.ofArray(bytes)); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppNonEmptyNoEntriesBytes() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "theta_non_empty_no_entries_cpp.sk"); + private static void deserializeNonEmptyNoEntriesFromFile(final GroupLanguage lang) { + final String fileName = "theta_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); final CompactThetaSketch sketch = CompactThetaSketch.wrap(bytes); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); diff --git a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java index 6c182baa2..8e40c9808 100644 --- a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java @@ -19,13 +19,13 @@ package org.apache.datasketches.tuple; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_HISTORICAL_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; -import static org.apache.datasketches.common.TestUtil.resPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_HISTORICAL_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -33,7 +33,8 @@ import java.lang.foreign.MemorySegment; import org.apache.datasketches.common.SketchesArgumentException; -import org.apache.datasketches.common.TestUtil; +import org.apache.datasketches.common.UtilityIO; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.tuple.adouble.DoubleSummary; import org.apache.datasketches.tuple.adouble.DoubleSummaryDeserializer; import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUnion; @@ -42,9 +43,57 @@ public class TupleCrossLanguageTest { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void generateForCppIntegerSummary() throws IOException { + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (int n: nArr) { + final UpdatableTupleSketch sk = + new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build(); + for (int i = 0; i < n; i++) { + sk.update(i, i); + } + putBytesToJavaPath("tuple_int_n" + n + "_java.sk", sk.compact().toByteArray()); + } + } + + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeTupleIntegerSummary(GroupLanguage.JAVA); + } + + @Test(groups = {CHECK_CPP_FILES}) + public void checkCpp() { + deserializeTupleIntegerSummary(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeTupleIntegerSummary(GroupLanguage.GO); + } + + private static void deserializeTupleIntegerSummary(final GroupLanguage lang) { + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (int n: nArr) { + final String fileName = "tuple_int_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue;} + //System.out.println(fileName); + final TupleSketch sketch = + TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer()); + assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); + assertTrue(n > 1000 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); + assertEquals(sketch.getEstimate(), n, n * 0.03); + final TupleSketchIterator it = sketch.iterator(); + while (it.next()) { + assertTrue(it.getHash() < sketch.getThetaLong()); + assertTrue(it.getSummary().getValue() < n); + } + } + } + @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) public void serialVersion1Compatibility() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "CompactSketchWithDoubleSummary4K_serialVersion1.sk"); + final byte[] byteArr = UtilityIO.getTestResourceBytes("CompactSketchWithDoubleSummary4K_serialVersion1.sk"); TupleSketch sketch = TupleSketch.heapifySketch(MemorySegment.ofArray(byteArr), new DoubleSummaryDeserializer()); Assert.assertTrue(sketch.isEstimationMode()); Assert.assertEquals(sketch.getEstimate(), 8192, 8192 * 0.99); @@ -60,7 +109,7 @@ public void serialVersion1Compatibility() { @Test(groups = {CHECK_CPP_HISTORICAL_FILES}) public void version2Compatibility() { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "TupleWithTestIntegerSummary4kTrimmedSerVer2.sk"); + final byte[] byteArr = UtilityIO.getTestResourceBytes("TupleWithTestIntegerSummary4kTrimmedSerVer2.sk"); TupleSketch sketch1 = TupleSketch.heapifySketch(MemorySegment.ofArray(byteArr), new IntegerSummaryDeserializer()); // construct the same way @@ -81,47 +130,28 @@ public void version2Compatibility() { Assert.assertEquals(sketch1.isEstimationMode(), sketch2.isEstimationMode()); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppIntegerSummary() throws IOException { - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "tuple_int_n" + n + "_cpp.sk"); - final TupleSketch sketch = - TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer()); - assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); - assertTrue(n > 1000 ? sketch.isEstimationMode() : !sketch.isEstimationMode()); - assertEquals(sketch.getEstimate(), n, n * 0.03); - final TupleSketchIterator it = sketch.iterator(); - while (it.next()) { - assertTrue(it.getHash() < sketch.getThetaLong()); - assertTrue(it.getSummary().getValue() < n); - } + @Test(expectedExceptions = RuntimeException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) + public void noSupportHeapifyV0_9_1() throws Exception { + final byte[] byteArr = UtilityIO.getTestResourceBytes("ArrayOfDoublesUnion_v0.9.1.sk"); + try { + ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(byteArr)); } - } - - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateForCppIntegerSummary() throws IOException { - final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; - for (int n: nArr) { - final UpdatableTupleSketch sk = - new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build(); - for (int i = 0; i < n; i++) { - sk.update(i, i); - } - putBytesToJavaPath("tuple_int_n" + n + "_java.sk", sk.compact().toByteArray()); + catch (final SketchesArgumentException e) { + throw new RuntimeException( + "EXPECTED EXCEPTION: Sketch Type mismatch. Expected ArrayOfDoublesUnion, got ArrayOfDoublesQuickSelectSketch"); } } - @Test(expectedExceptions = SketchesArgumentException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) - public void noSupportHeapifyV0_9_1() throws Exception { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "ArrayOfDoublesUnion_v0.9.1.sk"); - ArrayOfDoublesUnion.heapify(MemorySegment.ofArray(byteArr)); - } - - @Test(expectedExceptions = SketchesArgumentException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) + @Test(expectedExceptions = RuntimeException.class, groups = {CHECK_CPP_HISTORICAL_FILES}) public void noSupportWrapV0_9_1() throws Exception { - final byte[] byteArr = TestUtil.getFileBytes(resPath, "ArrayOfDoublesUnion_v0.9.1.sk"); - ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(byteArr)); + final byte[] byteArr = UtilityIO.getTestResourceBytes("ArrayOfDoublesUnion_v0.9.1.sk"); + try { + ArrayOfDoublesUnion.wrap(MemorySegment.ofArray(byteArr)); + } + catch (final SketchesArgumentException e) { + throw new RuntimeException( + "EXPECTED EXCEPTION: Sketch Type mismatch. Expected ArrayOfDoublesUnion, got ArrayOfDoublesQuickSelectSketch"); + } } } diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java index 121b8f52d..d19e0256c 100644 --- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/AodSketchCrossLanguageTest.java @@ -19,11 +19,12 @@ package org.apache.datasketches.tuple.arrayofdoubles; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.getFileBytes; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; @@ -31,6 +32,7 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.testng.annotations.Test; /** @@ -39,8 +41,8 @@ */ public class AodSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingOneValue() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeOneValue() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n: nArr) { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().build(); @@ -51,8 +53,8 @@ public void generateBinariesForCompatibilityTestingOneValue() throws IOException } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingThreeValues() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeThreeValues() throws IOException { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n: nArr) { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(3).build(); @@ -63,8 +65,8 @@ public void generateBinariesForCompatibilityTestingThreeValues() throws IOExcept } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeNonEmptyNoEntries() throws IOException { final ArrayOfDoublesUpdatableSketch sk = new ArrayOfDoublesUpdatableSketchBuilder().setSamplingProbability(0.01f).build(); sk.update(1, new double[] {1}); @@ -73,11 +75,34 @@ public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IO putBytesToJavaPath("aod_1_non_empty_no_entries_java.sk", sk.compact().toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeOneValue(GroupLanguage.JAVA); + deserializeThreeValues(GroupLanguage.JAVA); + deserializeOneValueNonEmptyNoEntries(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneValue() throws IOException { + public void checkCpp() { + deserializeOneValue(GroupLanguage.CPP); + deserializeThreeValues(GroupLanguage.CPP); + deserializeOneValueNonEmptyNoEntries(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeOneValue(GroupLanguage.GO); + deserializeThreeValues(GroupLanguage.GO); + deserializeOneValueNonEmptyNoEntries(GroupLanguage.GO); + } + + private static void deserializeOneValue(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "aod_1_n" + n + "_cpp.sk"); + final String fileName = "aod_1_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -89,11 +114,13 @@ public void deserializeFromCppOneValue() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppThreeValues() throws IOException { + private static void deserializeThreeValues(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; for (int n: nArr) { - final byte[] bytes = getFileBytes(cppPath, "aod_3_n" + n + "_cpp.sk"); + final String fileName = "aod_3_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); @@ -107,9 +134,11 @@ public void deserializeFromCppThreeValues() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneValueNonEmptyNoEntries() throws IOException { - final byte[] bytes = getFileBytes(cppPath, "aod_1_non_empty_no_entries_cpp.sk"); + private static void deserializeOneValueNonEmptyNoEntries(final GroupLanguage lang) { + final String fileName = "aod_1_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.err.println(fileName); final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketch.wrap(MemorySegment.ofArray(bytes)); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); diff --git a/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java index 8a1bf4187..40bfe057d 100644 --- a/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/strings/AosSketchCrossLanguageTest.java @@ -19,23 +19,25 @@ package org.apache.datasketches.tuple.strings; -import static org.apache.datasketches.common.TestUtil.CHECK_CPP_FILES; -import static org.apache.datasketches.common.TestUtil.GENERATE_JAVA_FILES; -import static org.apache.datasketches.common.TestUtil.cppPath; -import static org.apache.datasketches.common.TestUtil.putBytesToJavaPath; +import static org.apache.datasketches.common.UtilityIO.CHECK_CPP_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_GO_FILES; +import static org.apache.datasketches.common.UtilityIO.CHECK_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.GENERATE_JAVA_FILES; +import static org.apache.datasketches.common.UtilityIO.getFileBytes; +import static org.apache.datasketches.common.UtilityIO.putBytesToJavaPath; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import java.io.IOException; import java.lang.foreign.MemorySegment; -import java.nio.file.Files; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.datasketches.common.ResizeFactor; +import org.apache.datasketches.common.UtilityIO.GroupLanguage; import org.apache.datasketches.tuple.TupleSketch; import org.apache.datasketches.tuple.TupleSketchIterator; import org.testng.annotations.Test; @@ -46,8 +48,8 @@ */ public class AosSketchCrossLanguageTest { - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingOneString() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeOneString() throws IOException { int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); @@ -58,8 +60,8 @@ public void generateBinariesForCompatibilityTestingOneString() throws IOExceptio } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingThreeStrings() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeThreeStrings() throws IOException { int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); @@ -70,8 +72,8 @@ public void generateBinariesForCompatibilityTestingThreeStrings() throws IOExcep } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeOneStringNonEmptyNoEntries() throws IOException { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(12, ResizeFactor.X8, 0.01f); sk.update(new String[] {"key1"}, new String[] {"value1"}); @@ -80,8 +82,8 @@ public void generateBinariesForCompatibilityTestingNonEmptyNoEntries() throws IO putBytesToJavaPath("aos_1_non_empty_no_entries_java.sk", sk.compact().toByteArray()); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingMultiKeyStrings() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeMultiKeyStrings() throws IOException { int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); @@ -92,12 +94,16 @@ public void generateBinariesForCompatibilityTestingMultiKeyStrings() throws IOEx } } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingUnicodeStrings() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeUnicodeStrings() throws IOException { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); sk.update(new String[]{"키", "열쇠"}, new String[]{"밸류", "값"}); - sk.update(new String[]{"🔑", "🗝️"}, new String[]{"📦", "🎁"}); + //These are emojis that are outside the Basic Multilingual Plane and explicitly coded here + // as 16-bit surrogate pairs to fix a bug in the TestNG Eclipse Plugin (7.11.0). + // These 4 emojis are the Unicode Code Points (in order): + //"Key", U+1F511; "Old Key", U+1F5DD,U+FE0F; "Package", U+1F4E6; "Gift", U+1F381. + sk.update(new String[]{"\uD83D\uDD11", "\uD83D\uDDDD\uFE0F"}, new String[]{"\uD83D\uDCE6", "\uD83C\uDF81"}); sk.update(new String[]{"ключ1", "ключ2"}, new String[]{"ценить1", "ценить2"}); assertFalse(sk.isEmpty()); @@ -106,8 +112,8 @@ public void generateBinariesForCompatibilityTestingUnicodeStrings() throws IOExc putBytesToJavaPath("aos_unicode_java.sk", sk.compact().toByteArray()); } - @Test(groups = {GENERATE_JAVA_FILES}) - public void generateBinariesForCompatibilityTestingEmptyStrings() throws IOException { + @Test(groups = {GENERATE_JAVA_FILES}, priority = 0) + public void serializeEmptyStrings() throws IOException { ArrayOfStringsTupleSketch sk = new ArrayOfStringsTupleSketch(); sk.update(new String[]{""}, new String[]{"empty_key_value"}); @@ -120,12 +126,45 @@ public void generateBinariesForCompatibilityTestingEmptyStrings() throws IOExcep putBytesToJavaPath("aos_empty_strings_java.sk", sk.compact().toByteArray()); } + @Test(groups = {CHECK_JAVA_FILES}, priority = 1) + public void checkJava() { + deserializeOneString(GroupLanguage.JAVA); + deserializeFromThreeStrings(GroupLanguage.JAVA); + deserializeOneStringNonEmptyNoEntries(GroupLanguage.JAVA); + deserializeMultiKeyStrings(GroupLanguage.JAVA); + deserializeUnicodeStrings(GroupLanguage.JAVA); + deserializeEmptyStrings(GroupLanguage.JAVA); + } + @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneString() throws IOException { + public void checkCpp() { + deserializeOneString(GroupLanguage.CPP); + deserializeFromThreeStrings(GroupLanguage.CPP); + deserializeOneStringNonEmptyNoEntries(GroupLanguage.CPP); + deserializeMultiKeyStrings(GroupLanguage.CPP); + deserializeUnicodeStrings(GroupLanguage.CPP); + deserializeEmptyStrings(GroupLanguage.CPP); + } + + @Test(groups = {CHECK_GO_FILES}) + public void checkGo() { + deserializeOneString(GroupLanguage.GO); + deserializeFromThreeStrings(GroupLanguage.GO); + deserializeOneStringNonEmptyNoEntries(GroupLanguage.GO); + deserializeMultiKeyStrings(GroupLanguage.GO); + deserializeUnicodeStrings(GroupLanguage.GO); + deserializeEmptyStrings(GroupLanguage.GO); + } + + private static void deserializeOneString(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_1_n" + n + "_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + final String fileName = "aos_1_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertTrue(n > 1000? sketch.isEstimationMode() : !sketch.isEstimationMode()); @@ -139,12 +178,15 @@ public void deserializeFromCppOneString() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppThreeStrings() throws IOException { + private static void deserializeFromThreeStrings(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_3_n" + n + "_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + final String fileName = "aos_3_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertTrue(n > 1000? sketch.isEstimationMode() : !sketch.isEstimationMode()); @@ -158,21 +200,26 @@ public void deserializeFromCppThreeStrings() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppOneStringNonEmptyNoEntries() throws IOException { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_1_non_empty_no_entries_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); - + private static void deserializeOneStringNonEmptyNoEntries(final GroupLanguage lang) { + final String fileName = "aos_1_non_empty_no_entries" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertFalse(sketch.isEmpty()); assertEquals(sketch.getRetainedEntries(), 0); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppMultiKeyStrings() throws IOException { + private static void deserializeMultiKeyStrings(final GroupLanguage lang) { final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; for (int n : nArr) { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_multikey_n" + n + "_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + final String fileName = "aos_multikey_n" + n + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { continue; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty()); assertEquals(sketch.getEstimate(), n, n * 0.03); assertTrue(n > 1000? sketch.isEstimationMode() : !sketch.isEstimationMode()); @@ -186,10 +233,13 @@ public void deserializeFromCppMultiKeyStrings() throws IOException { } } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppUnicodeStrings() throws IOException { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_unicode_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + private static void deserializeUnicodeStrings(final GroupLanguage lang) { + final String fileName = "aos_unicode" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertFalse(sketch.isEmpty()); assertFalse(sketch.isEstimationMode()); assertEquals(sketch.getEstimate(), 3.0); @@ -200,10 +250,13 @@ public void deserializeFromCppUnicodeStrings() throws IOException { assertTrue(summaries.contains(Arrays.asList("ценить1", "ценить2"))); } - @Test(groups = {CHECK_CPP_FILES}) - public void deserializeFromCppEmptyStrings() throws IOException { - final byte[] bytes = Files.readAllBytes(cppPath.resolve("aos_empty_strings_cpp.sk")); - final TupleSketch sketch = ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); + private static void deserializeEmptyStrings(final GroupLanguage lang) { + final String fileName = "aos_empty_strings" + lang.sfx + ".sk"; + final byte[] bytes = getFileBytes(lang.pth, fileName); + if (bytes.length == 0) { return; } + //System.out.println(fileName); + final TupleSketch sketch = + ArrayOfStringsTupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new ArrayOfStringsSummaryDeserializer()); assertFalse(sketch.isEmpty()); assertFalse(sketch.isEstimationMode()); assertEquals(sketch.getEstimate(), 3.0); diff --git a/src/test/resources/testng.xml b/src/test/resources/testng.xml index a0e30835e..49d7e5c02 100644 --- a/src/test/resources/testng.xml +++ b/src/test/resources/testng.xml @@ -1,8 +1,8 @@ - + + + - - - - - + + + + + + +