Skip to content

Project: S3779 LZW ColGroup | ASML #2549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MasterBrain2000 wants to merge 9 commits into
base: pr-2420
Choose a base branch
from
Open

Project: S3779 LZW ColGroup | ASML #2549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
84f739b
PiecewiseLinearUtils.computeBreakpointSuccessive is compressed in O(n)
Mancer1 Jun 13, 2026
ed18dcd
Old DP algorithm and its usages are removed
Mancer1 Jun 15, 2026
9383091
add DDCLZW enum value to ColGroupType
Jun 15, 2026
5048313
new ColGroupPiecewiseLinearCompressed methods; fix testComputeSUm
m-ollka Jun 22, 2026
a649eb0
new replace & unitaryOperation
Jun 23, 2026
3df7122
ColGroupIO support
Jun 24, 2026
a294bf2
[Minor] fixed mistake in previous commit
Jul 7, 2026
fcb5fbc
Performance tests and Benchmarks for sucussive Piece wise algorithm a…
Mancer1 Jul 12, 2026
a02ed6f
added operation tests
Jul 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public abstract class AColGroup implements Serializable {
/** Public super types of compression ColGroups supported */
public static enum CompressionType {
UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCFOR, DDCFOR, DeltaDDC, DDCLZW, LinearFunctional,
PiecewiseLinear, PiecewiseLinearSuccessive;
PiecewiseLinearCompressed;

public boolean isDense() {
return this == DDC || this == CONST || this == DDCFOR || this == DDCFOR || this == DDCLZW;
Expand All @@ -88,7 +88,7 @@ public boolean isSDC() {
*/
protected static enum ColGroupType {
UNCOMPRESSED, RLE, OLE, DDC, CONST, EMPTY, SDC, SDCSingle, SDCSingleZeros, SDCZeros, SDCFOR, DDCFOR, DeltaDDC,
LinearFunctional, PiecewiseLinear;
LinearFunctional, PiecewiseLinearCompressed, DDCLZW;
}

/** The ColGroup indexes contained in the ColGroup */
Expand Down
27 changes: 3 additions & 24 deletions src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,12 +308,9 @@ else if(ct == CompressionType.LinearFunctional) {
return compressLinearFunctional(colIndexes, in, cs);
}
}
else if(ct == CompressionType.PiecewiseLinear) {
else if(ct == CompressionType.PiecewiseLinearCompressed) {
return compressPiecewiseLinearFunctional(colIndexes, in, cs);
}
else if(ct == CompressionType.PiecewiseLinearSuccessive) {
return compressPiecewiseLinearFunctionalSuccessive(colIndexes, in, cs);
}
else if(ct == CompressionType.DDCFOR) {
AColGroup g = directCompressDDC(colIndexes, cg);
if(g instanceof ColGroupDDC)
Expand Down Expand Up @@ -1110,7 +1107,7 @@ public static AColGroup compressPiecewiseLinearFunctional(IColIndex colIndexes,
for(int col = 0; col < numCols; col++) {
final int colIdx = colIndexes.get(col);
double[] column = PiecewiseLinearUtils.getColumn(in, colIdx);
PiecewiseLinearUtils.SegmentedRegression fit = PiecewiseLinearUtils.compressSegmentedLeastSquares(column,
PiecewiseLinearUtils.SegmentedRegression fit = PiecewiseLinearUtils.compressSuccessivePiecewiseLinear(column,
cs);
breakpointsPerCol[col] = fit.getBreakpoints();
interceptsPerCol[col] = fit.getIntercepts();
Expand All @@ -1124,25 +1121,7 @@ public static AColGroup compressPiecewiseLinearFunctional(IColIndex colIndexes,

public static AColGroup compressPiecewiseLinearFunctionalSuccessive(IColIndex colIndexes, MatrixBlock in,
CompressionSettings cs) {
final int numRows = in.getNumRows();
final int numCols = colIndexes.size();
int[][] breakpointsPerCol = new int[numCols][];
double[][] slopesPerCol = new double[numCols][];
double[][] interceptsPerCol = new double[numCols][];

for(int col = 0; col < numCols; col++) {
final int colIdx = colIndexes.get(col);
double[] column = PiecewiseLinearUtils.getColumn(in, colIdx);
PiecewiseLinearUtils.SegmentedRegression fit = PiecewiseLinearUtils.compressSuccessivePiecewiseLinear(
column, cs);
breakpointsPerCol[col] = fit.getBreakpoints();
interceptsPerCol[col] = fit.getIntercepts();
slopesPerCol[col] = fit.getSlopes();

}
return ColGroupPiecewiseLinearCompressed.create(colIndexes, breakpointsPerCol, slopesPerCol, interceptsPerCol,
numRows);

return compressPiecewiseLinearFunctional(colIndexes, in, cs);
}

private AColGroup compressSDCFromSparseTransposedBlock(IColIndex cols, int nrUniqueEstimate, double tupleSparsity) {
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ public static AColGroup readColGroup(DataInput in, int nRows) throws IOException
return ColGroupDeltaDDC.read(in);
case DDCLZW:
return ColGroupDDCLZW.read(in);
case PiecewiseLinearCompressed:
return ColGroupPiecewiseLinearCompressed.read(in);
case OLE:
return ColGroupOLE.read(in, nRows);
case RLE:
Expand Down
Loading