From 911d1c79ab4032c9d5de6ed4f9237beacd312cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 21 Jul 2026 12:42:09 +0800 Subject: [PATCH 1/2] [common] Classify idempotent aggregation functions Expose duplicate-application idempotence as aggregation metadata. Verify every aggregation implementation with a parameterized real-merger contract. --- .../fluss/metadata/AggFunctionType.java | 44 ++- .../AggregateRowMergerIdempotenceTest.java | 298 ++++++++++++++++++ 2 files changed, 328 insertions(+), 14 deletions(-) create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerIdempotenceTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/AggFunctionType.java b/fluss-common/src/main/java/org/apache/fluss/metadata/AggFunctionType.java index f0026fbba7..736f2cb0d9 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/AggFunctionType.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/AggFunctionType.java @@ -38,28 +38,44 @@ @PublicEvolving public enum AggFunctionType { // Numeric aggregation - SUM, - PRODUCT, - MAX, - MIN, + SUM(false), + PRODUCT(false), + MAX(true), + MIN(true), // Value selection - LAST_VALUE, - LAST_VALUE_IGNORE_NULLS, - FIRST_VALUE, - FIRST_VALUE_IGNORE_NULLS, + LAST_VALUE(true), + LAST_VALUE_IGNORE_NULLS(true), + FIRST_VALUE(true), + FIRST_VALUE_IGNORE_NULLS(true), // String aggregation - LISTAGG, - STRING_AGG, // Alias for LISTAGG - maps to same factory + LISTAGG(false), + STRING_AGG(false), // Alias for LISTAGG - maps to same factory // Boolean aggregation - BOOL_AND, - BOOL_OR, + BOOL_AND(true), + BOOL_OR(true), // Roaring bitmap aggregation - RBM32, - RBM64; + RBM32(true), + RBM64(true); + + private final boolean idempotent; + + AggFunctionType(boolean idempotent) { + this.idempotent = idempotent; + } + + /** + * Returns whether applying the same aggregation input again leaves the final materialized + * aggregate unchanged. + * + * @return true when duplicate aggregation input is idempotent + */ + public boolean isIdempotent() { + return idempotent; + } // ------------------------------------------------------------------------------------------ diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerIdempotenceTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerIdempotenceTest.java new file mode 100644 index 0000000000..61801aa53c --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/AggregateRowMergerIdempotenceTest.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv.rowmerger; + +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.metadata.AggFunctionType; +import org.apache.fluss.metadata.AggFunctions; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.record.BinaryValue; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.BinaryRow; +import org.apache.fluss.server.utils.RoaringBitmapUtils; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.longlong.Roaring64Bitmap; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; + +/** Contract tests between server aggregation semantics and idempotence classification. */ +class AggregateRowMergerIdempotenceTest { + + private static final short SCHEMA_ID = (short) 1; + + private static final SemanticComparator LONG_EQUALITY = + (left, right) -> + haveSameNullState(left, right) + && (left.isNullAt(1) || left.getLong(1) == right.getLong(1)); + + private static final SemanticComparator INT_EQUALITY = + (left, right) -> + haveSameNullState(left, right) + && (left.isNullAt(1) || left.getInt(1) == right.getInt(1)); + + private static final SemanticComparator STRING_EQUALITY = + (left, right) -> + haveSameNullState(left, right) + && (left.isNullAt(1) || left.getString(1).equals(right.getString(1))); + + private static final SemanticComparator BOOLEAN_EQUALITY = + (left, right) -> + haveSameNullState(left, right) + && (left.isNullAt(1) || left.getBoolean(1) == right.getBoolean(1)); + + @ParameterizedTest(name = "{0}") + @MethodSource("aggregationIdempotenceCases") + void testAggregationSemanticsMatchIdempotenceClassification( + AggFunctionType functionType, + Schema schema, + Configuration configuration, + Object committedValue, + List dirtySequence, + SemanticComparator semanticComparator) + throws IOException { + AggregateRowMerger merger = createMerger(schema, configuration); + BinaryValue committed = value(schema, committedValue); + + BinaryValue once = apply(merger, schema, committed, dirtySequence); + BinaryValue twice = apply(merger, schema, once, dirtySequence); + + assertThat(semanticComparator.areEqual(once.row, twice.row)) + .as("semantic idempotence for %s", functionType) + .isEqualTo(functionType.isIdempotent()); + } + + @Test + void testContractCasesCoverEveryAggregationFunction() throws IOException { + Set coveredTypes = + aggregationIdempotenceCases() + .map(arguments -> (AggFunctionType) arguments.get()[0]) + .collect( + Collectors.toCollection( + () -> EnumSet.noneOf(AggFunctionType.class))); + + assertThat(coveredTypes).containsExactlyInAnyOrder(AggFunctionType.values()); + } + + static Stream aggregationIdempotenceCases() throws IOException { + return Stream.of( + Arguments.of( + AggFunctionType.SUM, + schema(AggFunctionType.SUM, DataTypes.BIGINT()), + new Configuration(), + 10L, + Arrays.asList(2L, 3L), + LONG_EQUALITY), + Arguments.of( + AggFunctionType.PRODUCT, + schema(AggFunctionType.PRODUCT, DataTypes.BIGINT()), + new Configuration(), + 2L, + Arrays.asList(3L, 5L), + LONG_EQUALITY), + Arguments.of( + AggFunctionType.MAX, + schema(AggFunctionType.MAX, DataTypes.INT()), + new Configuration(), + 10, + Arrays.asList(8, 20, 15), + INT_EQUALITY), + Arguments.of( + AggFunctionType.MIN, + schema(AggFunctionType.MIN, DataTypes.INT()), + new Configuration(), + 10, + Arrays.asList(12, 3, 7), + INT_EQUALITY), + Arguments.of( + AggFunctionType.LAST_VALUE, + schema(AggFunctionType.LAST_VALUE, DataTypes.STRING()), + new Configuration(), + "committed", + Arrays.asList("tail", null), + STRING_EQUALITY), + Arguments.of( + AggFunctionType.LAST_VALUE_IGNORE_NULLS, + schema(AggFunctionType.LAST_VALUE_IGNORE_NULLS, DataTypes.STRING()), + new Configuration(), + "committed", + Arrays.asList(null, "tail", null), + STRING_EQUALITY), + Arguments.of( + AggFunctionType.FIRST_VALUE, + schema(AggFunctionType.FIRST_VALUE, DataTypes.STRING()), + new Configuration(), + null, + Arrays.asList("later", null), + STRING_EQUALITY), + Arguments.of( + AggFunctionType.FIRST_VALUE_IGNORE_NULLS, + schema(AggFunctionType.FIRST_VALUE_IGNORE_NULLS, DataTypes.STRING()), + new Configuration(), + null, + Arrays.asList(null, "first", "later"), + STRING_EQUALITY), + Arguments.of( + AggFunctionType.LISTAGG, + schema( + AggFunctionType.LISTAGG, + DataTypes.STRING(), + Collections.singletonMap(AggFunctions.PARAM_DELIMITER, "|")), + new Configuration(), + "committed", + Arrays.asList("dirty-1", "dirty-2"), + STRING_EQUALITY), + Arguments.of( + AggFunctionType.STRING_AGG, + schema( + AggFunctionType.STRING_AGG, + DataTypes.STRING(), + Collections.singletonMap(AggFunctions.PARAM_DELIMITER, ";")), + new Configuration(), + "committed", + Arrays.asList("dirty-1", "dirty-2"), + STRING_EQUALITY), + Arguments.of( + AggFunctionType.BOOL_AND, + schema(AggFunctionType.BOOL_AND, DataTypes.BOOLEAN()), + new Configuration(), + true, + Arrays.asList(true, false, true), + BOOLEAN_EQUALITY), + Arguments.of( + AggFunctionType.BOOL_OR, + schema(AggFunctionType.BOOL_OR, DataTypes.BOOLEAN()), + new Configuration(), + false, + Arrays.asList(false, true, false), + BOOLEAN_EQUALITY), + Arguments.of( + AggFunctionType.RBM32, + schema(AggFunctionType.RBM32, DataTypes.BYTES()), + new Configuration(), + bitmap32(1, 2), + Arrays.asList(bitmap32(2, 3), bitmap32(4)), + (SemanticComparator) AggregateRowMergerIdempotenceTest::haveEqualRbm32Sets), + Arguments.of( + AggFunctionType.RBM64, + schema(AggFunctionType.RBM64, DataTypes.BYTES()), + new Configuration(), + bitmap64(10L, 20L), + Arrays.asList(bitmap64(20L, 30L), bitmap64(40L)), + (SemanticComparator) + AggregateRowMergerIdempotenceTest::haveEqualRbm64Sets)); + } + + private static AggregateRowMerger createMerger(Schema schema, Configuration configuration) { + TableConfig tableConfig = new TableConfig(configuration); + TestingSchemaGetter schemaGetter = + new TestingSchemaGetter(new SchemaInfo(schema, SCHEMA_ID)); + AggregateRowMerger merger = + new AggregateRowMerger(tableConfig, tableConfig.getKvFormat(), schemaGetter); + merger.configureTargetColumns(null, SCHEMA_ID, schema); + return merger; + } + + private static BinaryValue apply( + AggregateRowMerger merger, + Schema schema, + BinaryValue committed, + List dirtySequence) { + BinaryValue result = committed; + for (Object dirtyValue : dirtySequence) { + result = merger.merge(result, value(schema, dirtyValue)); + } + return result; + } + + private static BinaryValue value(Schema schema, Object aggregateValue) { + BinaryRow row = compactedRow(schema.getRowType(), new Object[] {1, aggregateValue}); + return new BinaryValue(SCHEMA_ID, row); + } + + private static Schema schema(AggFunctionType functionType, DataType dataType) { + return schema(functionType, dataType, Collections.emptyMap()); + } + + private static Schema schema( + AggFunctionType functionType, DataType dataType, Map parameters) { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("value", dataType, AggFunctions.of(functionType, parameters)) + .primaryKey("id") + .build(); + } + + private static boolean haveSameNullState(BinaryRow left, BinaryRow right) { + return left.isNullAt(1) == right.isNullAt(1); + } + + private static boolean haveEqualRbm32Sets(BinaryRow left, BinaryRow right) throws IOException { + RoaringBitmap leftBitmap = new RoaringBitmap(); + RoaringBitmap rightBitmap = new RoaringBitmap(); + RoaringBitmapUtils.deserializeRoaringBitmap32(leftBitmap, left.getBytes(1)); + RoaringBitmapUtils.deserializeRoaringBitmap32(rightBitmap, right.getBytes(1)); + return leftBitmap.equals(rightBitmap); + } + + private static boolean haveEqualRbm64Sets(BinaryRow left, BinaryRow right) throws IOException { + Roaring64Bitmap leftBitmap = new Roaring64Bitmap(); + Roaring64Bitmap rightBitmap = new Roaring64Bitmap(); + RoaringBitmapUtils.deserializeRoaringBitmap64(leftBitmap, left.getBytes(1)); + RoaringBitmapUtils.deserializeRoaringBitmap64(rightBitmap, right.getBytes(1)); + return leftBitmap.equals(rightBitmap); + } + + private static byte[] bitmap32(int... values) throws IOException { + RoaringBitmap bitmap = new RoaringBitmap(); + bitmap.add(values); + return RoaringBitmapUtils.serializeRoaringBitmap32(bitmap); + } + + private static byte[] bitmap64(long... values) throws IOException { + Roaring64Bitmap bitmap = new Roaring64Bitmap(); + for (long value : values) { + bitmap.add(value); + } + return RoaringBitmapUtils.serializeRoaringBitmap64(bitmap); + } + + @FunctionalInterface + private interface SemanticComparator { + boolean areEqual(BinaryRow left, BinaryRow right) throws IOException; + } +} From c55d63cf3573a26646de5e9cb35cd928bfae5cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Tue, 21 Jul 2026 12:42:29 +0800 Subject: [PATCH 2/2] [flink] Skip unnecessary aggregation undo recovery Use no-op recovery when all effective aggregation operations are idempotent, and retain undo recovery otherwise. Preserve operator topology and checkpoint compatibility, clean stale recovery metadata, and cover SQL and DataStream failover paths. --- .../flink/catalog/FlinkTableFactory.java | 4 + .../sink/AggregationRecoveryDecider.java | 86 ++ .../apache/fluss/flink/sink/FlinkSink.java | 17 +- .../fluss/flink/sink/FlinkTableSink.java | 100 ++- .../fluss/flink/sink/FlussSinkBuilder.java | 23 +- .../fluss/flink/sink/undo/RecoveryAction.java | 36 + .../flink/sink/undo/UndoRecoveryOperator.java | 281 +++--- .../undo/UndoRecoveryOperatorFactory.java | 78 +- .../flink/sink/writer/UpsertSinkWriter.java | 3 +- .../sink/AggregationRecoveryDeciderTest.java | 116 +++ .../flink/sink/FlinkSinkTopologyTest.java | 117 +++ .../fluss/flink/sink/FlinkTableSinkTest.java | 152 ++++ .../flink/sink/FlussSinkBuilderTest.java | 56 ++ .../fluss/flink/sink/UndoRecoveryITCase.java | 801 +++++++++--------- .../sink/testutils/FailingCountingSource.java | 607 ++++++------- .../sink/undo/UndoRecoveryOperatorITCase.java | 214 +++++ .../sink/undo/UndoRecoveryOperatorTest.java | 266 ++++++ website/docs/engine-flink/writes.md | 37 +- 18 files changed, 2083 insertions(+), 911 deletions(-) create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/AggregationRecoveryDecider.java create mode 100644 fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/RecoveryAction.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/AggregationRecoveryDeciderTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkSinkTopologyTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkTest.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorITCase.java create mode 100644 fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorTest.java diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java index ab64959abd..836cdafe6c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java @@ -30,6 +30,7 @@ import org.apache.fluss.flink.source.FlinkTableSource; import org.apache.fluss.flink.source.reader.LeaseContext; import org.apache.fluss.flink.utils.FlinkConnectorOptionsUtils; +import org.apache.fluss.flink.utils.FlinkConversions; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.TablePath; @@ -185,6 +186,8 @@ public DynamicTableSink createDynamicTableSink(Context context) { List partitionKeys = resolvedCatalogTable.getPartitionKeys(); RowType rowType = (RowType) context.getPhysicalRowDataType().getLogicalType(); + org.apache.fluss.metadata.Schema tableSchema = + FlinkConversions.toFlussTable(resolvedCatalogTable).getSchema(); MergeEngineType mergeEngineType = tableOptions.get(toFlinkOption(ConfigOptions.TABLE_MERGE_ENGINE)); @@ -203,6 +206,7 @@ public DynamicTableSink createDynamicTableSink(Context context) { toFlussClientConfig( context.getCatalogTable().getOptions(), context.getConfiguration()), rowType, + tableSchema, primaryKeyIndexes, partitionKeys, isStreamingMode, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/AggregationRecoveryDecider.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/AggregationRecoveryDecider.java new file mode 100644 index 0000000000..e706ef6768 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/AggregationRecoveryDecider.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink; + +import org.apache.fluss.flink.sink.undo.RecoveryAction; +import org.apache.fluss.metadata.AggFunctionType; +import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.Schema; + +import javax.annotation.Nullable; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Decides the failover correction action for an aggregation sink. */ +final class AggregationRecoveryDecider { + + private AggregationRecoveryDecider() {} + + static RecoveryAction decide( + Schema tableSchema, + @Nullable int[] targetColumnIndexes, + DeleteBehavior deleteBehavior, + boolean ignoreDelete, + boolean inputKnownUpsertOnly) { + checkNotNull(deleteBehavior, "deleteBehavior must not be null."); + checkNotNull(tableSchema, "tableSchema must not be null."); + + List columns = tableSchema.getColumns(); + int[] effectiveTargets = + targetColumnIndexes == null + ? IntStream.range(0, columns.size()).toArray() + : targetColumnIndexes; + for (int targetIndex : effectiveTargets) { + checkArgument( + targetIndex >= 0 && targetIndex < columns.size(), + "Invalid target column index %s for schema with %s columns.", + targetIndex, + columns.size()); + } + + if (!ignoreDelete && deleteBehavior == DeleteBehavior.ALLOW && !inputKnownUpsertOnly) { + return RecoveryAction.UNDO; + } + + Set primaryKeyIndexes = new HashSet<>(); + for (int primaryKeyIndex : tableSchema.getPrimaryKeyIndexes()) { + primaryKeyIndexes.add(primaryKeyIndex); + } + for (int targetIndex : effectiveTargets) { + if (primaryKeyIndexes.contains(targetIndex)) { + continue; + } + + AggFunctionType functionType = + columns.get(targetIndex) + .getAggFunction() + .map(function -> function.getType()) + .orElse(AggFunctionType.LAST_VALUE_IGNORE_NULLS); + if (!functionType.isIdempotent()) { + return RecoveryAction.UNDO; + } + } + return RecoveryAction.NO_OP; + } +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkSink.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkSink.java index 159b42764c..35964cf20c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkSink.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkSink.java @@ -27,6 +27,7 @@ import org.apache.fluss.flink.sink.shuffle.StatisticsOrRecordChannelComputer; import org.apache.fluss.flink.sink.shuffle.StatisticsOrRecordTypeInformation; import org.apache.fluss.flink.sink.undo.ProducerOffsetReporter; +import org.apache.fluss.flink.sink.undo.RecoveryAction; import org.apache.fluss.flink.sink.undo.UndoRecoveryOperatorFactory; import org.apache.fluss.flink.sink.writer.AppendSinkWriter; import org.apache.fluss.flink.sink.writer.FlinkSinkWriter; @@ -250,14 +251,15 @@ static class UpsertSinkWriterBuilder private final DistributionMode distributionMode; private final FlussSerializationSchema flussSerializationSchema; private final boolean enableUndoRecovery; + private final RecoveryAction recoveryAction; @Nullable private final String producerId; /** * Optional context for reporting offsets to the upstream UndoRecoveryOperator. * - *

This is set internally by {@link #addPreWriteTopology} when UndoRecoveryOperator is - * added to the pipeline. The context is then passed to the UpsertSinkWriter during - * creation. + *

This is set internally by {@link #addPreWriteTopology} only when the recovery action + * is {@link RecoveryAction#UNDO}. The NO_OP compatibility transform remains in the topology + * without installing a writer callback. * *

Note: This field is NOT transient because the ProducerOffsetReporterHolder is * serializable and needs to survive job serialization to be passed to the TaskManager. @@ -276,6 +278,7 @@ static class UpsertSinkWriterBuilder DistributionMode distributionMode, FlussSerializationSchema flussSerializationSchema, boolean enableUndoRecovery, + RecoveryAction recoveryAction, @Nullable String producerId) { this.tablePath = tablePath; this.flussConfig = flussConfig; @@ -288,6 +291,7 @@ static class UpsertSinkWriterBuilder this.distributionMode = distributionMode; this.flussSerializationSchema = flussSerializationSchema; this.enableUndoRecovery = enableUndoRecovery; + this.recoveryAction = recoveryAction; this.producerId = producerId; } @@ -341,7 +345,8 @@ public DataStream addPreWriteTopology(DataStream input) { targetColumnIndexes, numBucket, !partitionKeys.isEmpty(), - producerId); + producerId, + recoveryAction); stream = stream.transform( @@ -350,7 +355,9 @@ public DataStream addPreWriteTopology(DataStream input) { operatorFactory) .setParallelism(stream.getParallelism()); - offsetReporter = operatorFactory.getProducerOffsetReporter(); + if (operatorFactory.getRecoveryAction() == RecoveryAction.UNDO) { + offsetReporter = operatorFactory.getProducerOffsetReporter(); + } } return stream; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkTableSink.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkTableSink.java index 8035028dec..b185d5bc9a 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkTableSink.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkTableSink.java @@ -17,9 +17,11 @@ package org.apache.fluss.flink.sink; +import org.apache.fluss.annotation.Internal; import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.sink.serializer.RowDataSerializationSchema; import org.apache.fluss.flink.sink.shuffle.DistributionMode; +import org.apache.fluss.flink.sink.undo.RecoveryAction; import org.apache.fluss.flink.sink.writer.FlinkSinkWriter; import org.apache.fluss.flink.utils.PushdownUtils; import org.apache.fluss.flink.utils.PushdownUtils.FieldEqual; @@ -27,6 +29,7 @@ import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.DeleteBehavior; import org.apache.fluss.metadata.MergeEngineType; +import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.row.GenericRow; @@ -65,6 +68,7 @@ import static org.apache.fluss.flink.utils.PushdownUtils.extractFieldEquals; /** A Flink {@link DynamicTableSink}. */ +@Internal public class FlinkTableSink implements DynamicTableSink, SupportsPartitioning, @@ -75,6 +79,7 @@ public class FlinkTableSink private final TablePath tablePath; private final Configuration flussConfig; private final RowType tableRowType; + @Nullable private final Schema tableSchema; private final int[] primaryKeyIndexes; private final List partitionKeys; private final boolean streaming; @@ -87,9 +92,13 @@ public class FlinkTableSink private final @Nullable DataLakeFormat lakeFormat; @Nullable private final String producerId; + private boolean inputChangelogModeKnown; + private boolean inputKnownUpsertOnly; + private boolean appliedUpdates = false; @Nullable private GenericRow deleteRow; + /** Creates a table sink without aggregation metadata, retaining conservative recovery. */ public FlinkTableSink( TablePath tablePath, Configuration flussConfig, @@ -105,9 +114,45 @@ public FlinkTableSink( List bucketKeys, DistributionMode distributionMode, @Nullable String producerId) { + this( + tablePath, + flussConfig, + tableRowType, + null, + primaryKeyIndexes, + partitionKeys, + streaming, + mergeEngineType, + lakeFormat, + sinkIgnoreDelete, + tableDeleteBehavior, + numBucket, + bucketKeys, + distributionMode, + producerId); + } + + /** Creates a table sink with Fluss aggregation metadata for recovery classification. */ + public FlinkTableSink( + TablePath tablePath, + Configuration flussConfig, + RowType tableRowType, + @Nullable Schema tableSchema, + int[] primaryKeyIndexes, + List partitionKeys, + boolean streaming, + @Nullable MergeEngineType mergeEngineType, + @Nullable DataLakeFormat lakeFormat, + boolean sinkIgnoreDelete, + DeleteBehavior tableDeleteBehavior, + int numBucket, + List bucketKeys, + DistributionMode distributionMode, + @Nullable String producerId) { this.tablePath = tablePath; this.flussConfig = flussConfig; this.tableRowType = tableRowType; + this.tableSchema = tableSchema; this.primaryKeyIndexes = primaryKeyIndexes; this.partitionKeys = partitionKeys; this.streaming = streaming; @@ -123,23 +168,34 @@ public FlinkTableSink( @Override public ChangelogMode getChangelogMode(ChangelogMode requestedMode) { + ChangelogMode acceptedMode; if (!streaming) { - return ChangelogMode.insertOnly(); - } else { - if (primaryKeyIndexes.length > 0 || sinkIgnoreDelete) { - // primary-key table or ignore_delete mode can accept RowKind.DELETE - ChangelogMode.Builder builder = ChangelogMode.newBuilder(); - for (RowKind kind : requestedMode.getContainedKinds()) { - // optimize out the update_before messages - if (kind != RowKind.UPDATE_BEFORE) { - builder.addContainedKind(kind); - } + acceptedMode = ChangelogMode.insertOnly(); + } else if (primaryKeyIndexes.length > 0 || sinkIgnoreDelete) { + ChangelogMode.Builder builder = ChangelogMode.newBuilder(); + for (RowKind kind : requestedMode.getContainedKinds()) { + if (kind != RowKind.UPDATE_BEFORE) { + builder.addContainedKind(kind); } - return builder.build(); - } else { - return ChangelogMode.insertOnly(); } + acceptedMode = builder.build(); + } else { + acceptedMode = ChangelogMode.insertOnly(); } + + // The planner may invoke this method multiple times, including with generic capability + // probes. Since the public API does not identify the final negotiation, only retain the + // upsert-only proof when every observed accepted mode provides it. + boolean acceptedModeKnownUpsertOnly = + !acceptedMode.getContainedKinds().contains(RowKind.DELETE) + && !acceptedMode.getContainedKinds().contains(RowKind.UPDATE_BEFORE); + if (inputChangelogModeKnown) { + inputKnownUpsertOnly &= acceptedModeKnownUpsertOnly; + } else { + inputChangelogModeKnown = true; + inputKnownUpsertOnly = acceptedModeKnownUpsertOnly; + } + return acceptedMode; } @Override @@ -206,9 +262,21 @@ public DataStreamSink consumeDataStream( }; } + RecoveryAction getRecoveryAction(@Nullable int[] targetColumnIndexes) { + if (mergeEngineType != MergeEngineType.AGGREGATION || tableSchema == null) { + return RecoveryAction.UNDO; + } + return AggregationRecoveryDecider.decide( + tableSchema, + targetColumnIndexes, + tableDeleteBehavior, + sinkIgnoreDelete, + inputKnownUpsertOnly); + } + private FlinkSink getFlinkSink(int[] targetColumnIndexes) { - // Enable undo recovery for aggregation tables boolean enableUndoRecovery = mergeEngineType == MergeEngineType.AGGREGATION; + RecoveryAction recoveryAction = getRecoveryAction(targetColumnIndexes); FlinkSink.SinkWriterBuilder flinkSinkWriterBuilder = (primaryKeyIndexes.length > 0) @@ -224,6 +292,7 @@ private FlinkSink getFlinkSink(int[] targetColumnIndexes) { distributionMode, new RowDataSerializationSchema(false, sinkIgnoreDelete), enableUndoRecovery, + recoveryAction, producerId) : new FlinkSink.AppendSinkWriterBuilder<>( tablePath, @@ -254,6 +323,7 @@ public DynamicTableSink copy() { tablePath, flussConfig, tableRowType, + tableSchema, primaryKeyIndexes, partitionKeys, streaming, @@ -265,6 +335,8 @@ public DynamicTableSink copy() { bucketKeys, distributionMode, producerId); + sink.inputChangelogModeKnown = inputChangelogModeKnown; + sink.inputKnownUpsertOnly = inputKnownUpsertOnly; sink.appliedUpdates = appliedUpdates; sink.deleteRow = deleteRow; return sink; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlussSinkBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlussSinkBuilder.java index c8808f12f8..288dc22d7c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlussSinkBuilder.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlussSinkBuilder.java @@ -25,8 +25,10 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.sink.serializer.FlussSerializationSchema; import org.apache.fluss.flink.sink.shuffle.DistributionMode; +import org.apache.fluss.flink.sink.undo.RecoveryAction; import org.apache.fluss.flink.sink.writer.FlinkSinkWriter; import org.apache.fluss.metadata.DataLakeFormat; +import org.apache.fluss.metadata.DeleteBehavior; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; @@ -35,6 +37,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -209,7 +213,8 @@ public FlussSink build() { boolean isUpsert = tableInfo.hasPrimaryKey(); - // Detect if this is an aggregation table that needs undo recovery + // Keep the recovery compatibility transform for every aggregation table. The selected + // action determines whether it performs undo recovery or acts as a no-op shell. MergeEngineType mergeEngineType = tableInfo.getTableConfig().getMergeEngineType().orElse(null); boolean enableUndoRecovery = mergeEngineType == MergeEngineType.AGGREGATION; @@ -224,6 +229,7 @@ public FlussSink build() { tableRowType.getFieldNames(), tableInfo.getPrimaryKeys(), partialUpdateColumns); + RecoveryAction recoveryAction = determineRecoveryAction(tableInfo, targetColumnIndexes); writerBuilder = new FlinkSink.UpsertSinkWriterBuilder<>( tablePath, @@ -237,6 +243,7 @@ public FlussSink build() { distributionMode, serializationSchema, enableUndoRecovery, + recoveryAction, producerId); } else { LOG.info("Initializing Fluss append sink writer ..."); @@ -268,6 +275,20 @@ private void validateConfiguration() { } // -------------- Test-visible helper methods -------------- + static RecoveryAction determineRecoveryAction( + TableInfo tableInfo, @Nullable int[] targetColumnIndexes) { + MergeEngineType mergeEngineType = + tableInfo.getTableConfig().getMergeEngineType().orElse(null); + if (mergeEngineType != MergeEngineType.AGGREGATION) { + return RecoveryAction.UNDO; + } + + DeleteBehavior deleteBehavior = + tableInfo.getTableConfig().getDeleteBehavior().orElse(DeleteBehavior.ALLOW); + return AggregationRecoveryDecider.decide( + tableInfo.getSchema(), targetColumnIndexes, deleteBehavior, false, false); + } + /** * Computes target column indexes for partial updates. If {@code specifiedColumns} is null or * empty, returns null indicating full update. Validates that all primary key columns are diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/RecoveryAction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/RecoveryAction.java new file mode 100644 index 0000000000..9099841efa --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/RecoveryAction.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink.undo; + +import org.apache.fluss.annotation.Internal; + +/** + * Failover correction action for an aggregation sink. + * + *

The action is fixed when the sink topology is built. Checkpoints record the selected action; + * restoring a NO_OP checkpoint as UNDO is unsupported because NO_OP checkpoints contain no bucket + * offset baseline. + */ +@Internal +public enum RecoveryAction { + /** Keep the current Fluss state without tracking or applying undo recovery. */ + NO_OP, + + /** Restore the checkpoint materialized state through the existing undo-recovery path. */ + UNDO +} diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperator.java index da055f1f56..f0135417f9 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperator.java @@ -32,6 +32,7 @@ import org.apache.flink.api.common.state.ListState; import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.runtime.state.StateInitializationContext; import org.apache.flink.runtime.state.StateSnapshotContext; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; @@ -47,35 +48,29 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + /** - * A Flink stream operator that manages undo recovery state using Union List State. - * - *

This operator ensures correct state redistribution during scale up/down scenarios by using - * Union List State instead of the default List State. During recovery, each subtask receives a - * complete copy of all states from all previous subtasks, then uses {@link RecoveryOffsetManager} - * to determine the recovery strategy. + * A Flink stream operator that preserves aggregation failover-recovery topology and state identity. * - *

The operator performs the following functions: + *

With {@link RecoveryAction#UNDO}, the operator tracks bucket offsets in Union List State and + * uses {@link RecoveryOffsetManager} and {@link UndoRecoveryManager} to restore the checkpointed + * materialized state. * - *

    - *
  • Manages Union List State for bucket offsets - *
  • Uses {@link RecoveryOffsetManager} to determine recovery strategy (checkpoint or producer - * offsets) - *
  • Executes undo recovery during {@code initializeState()} using {@link UndoRecoveryManager} - *
  • Receives offset reports from downstream Writer via {@link ProducerOffsetReporter} - *
  • Snapshots state during checkpoints - *
  • Cleans up producer offsets after first checkpoint (Task0 only) - *
  • Passes through input elements unchanged - *
+ *

With {@link RecoveryAction#NO_OP}, the operator remains in the topology for checkpoint + * compatibility and skips data undo and offset tracking. When a producer ID is configured, subtask + * 0 uses a short-lived connection to delete its stale producer offsets once during initialization; + * otherwise NO_OP does not connect. No table is opened. It still claims the existing undo state + * descriptor so that the next checkpoint can discard that state. A separate constant-size marker + * records which action produced a checkpoint; a checkpoint produced by NO_OP cannot later be + * restored with UNDO because it has no undo baseline. * - *

Recovery Strategy: The operator uses {@link - * RecoveryOffsetManager#determineRecoveryStrategy} to decide whether to use checkpoint state or - * producer offsets for recovery. This handles both normal checkpoint recovery and pre-checkpoint - * failure scenarios. + *

Input elements always pass through unchanged. * * @param The type of input elements * @see ProducerOffsetReporter @@ -92,6 +87,9 @@ public class UndoRecoveryOperator extends AbstractStreamOperator /** State descriptor name for the Union List State. */ private static final String UNDO_RECOVERY_STATE_NAME = "undo_recovery_state"; + /** State descriptor name for the recovery action that produced a checkpoint. */ + private static final String RECOVERY_ACTION_STATE_NAME = "recovery_action_state"; + // ==================== Configuration Fields ==================== /** The table path for the Fluss table. */ @@ -116,15 +114,18 @@ public class UndoRecoveryOperator extends AbstractStreamOperator * The producer ID used for producer offset snapshot management. * *

This is used by {@link RecoveryOffsetManager} to register and retrieve producer offsets - * for pre-checkpoint failure recovery. If null at construction time, it will be resolved to the - * Flink job ID during initialization. + * for pre-checkpoint failure recovery. If null, UNDO resolves it to the Flink job ID while + * NO_OP performs no producer-offset operation. */ @Nullable private final String configuredProducerId; + private final RecoveryAction recoveryAction; + /** * The resolved producer ID (either configured or defaulted to Flink job ID). * - *

This is set during {@link #initializeState} and used for all producer offset operations. + *

This is set during {@link #initializeState} for UNDO subtasks and for NO_OP subtask 0 when + * a producer ID is configured, and used for producer offset operations. */ private transient String resolvedProducerId; @@ -144,14 +145,18 @@ public class UndoRecoveryOperator extends AbstractStreamOperator // ==================== State Fields ==================== - /** Union List State for storing bucket offsets across checkpoints. */ + /** Union List State used by UNDO and claimed for cleanup by the NO_OP compatibility shell. */ private transient ListState undoStateList; + /** Union List State containing the action that produced a checkpoint. */ + private transient ListState recoveryActionState; + /** * Map from TableBucket to the latest written offset. * - *

This map is updated by the downstream SinkWriter via {@link #reportOffset(TableBucket, - * long)} and is used to create WriterState during checkpoint snapshotting. + *

For UNDO, this map is updated by the downstream SinkWriter via {@link + * #reportOffset(TableBucket, long)} and is used to create WriterState during checkpoint + * snapshotting. It remains empty for NO_OP. * *

Uses ConcurrentHashMap for thread-safe updates from async write callbacks. The * ConcurrentHashMap's native thread-safety is sufficient since {@code merge()} is atomic and @@ -188,27 +193,7 @@ public class UndoRecoveryOperator extends AbstractStreamOperator // ==================== Constructor ==================== - /** - * Creates a new UndoRecoveryOperator with StreamOperatorParameters. - * - *

This constructor is used by {@link UndoRecoveryOperatorFactory} to create the operator - * instance. It calls {@link #setup} internally to properly initialize the operator with Flink's - * runtime context. - * - * @param parameters the stream operator parameters from Flink runtime - * @param tablePath the table path for the Fluss table - * @param flussConfig the Fluss configuration - * @param tableRowType the row type of the table - * @param targetColumnIndexes target column indexes for partial update (null for full row) - * @param numBuckets the number of buckets in the table - * @param isPartitioned whether the table is partitioned - * @param producerId the producer ID for producer offset management (null to use Flink job ID) - * @param producerOffsetsPollIntervalMs the polling interval for producer offsets - * @param maxPollTimeoutMs the maximum total time to poll for producer offsets - * @param offsetReporterRegistryId the registry ID for registering/removing in the delegate - * registry - */ - public UndoRecoveryOperator( + UndoRecoveryOperator( StreamOperatorParameters parameters, TablePath tablePath, Configuration flussConfig, @@ -217,6 +202,7 @@ public UndoRecoveryOperator( int numBuckets, boolean isPartitioned, @Nullable String producerId, + RecoveryAction recoveryAction, long producerOffsetsPollIntervalMs, long maxPollTimeoutMs, String offsetReporterRegistryId) { @@ -227,13 +213,12 @@ public UndoRecoveryOperator( this.targetColumnIndexes = targetColumnIndexes; this.numBuckets = numBuckets; this.isPartitioned = isPartitioned; - this.configuredProducerId = - producerId; // May be null, will be resolved in initializeState() + this.configuredProducerId = producerId; + this.recoveryAction = checkNotNull(recoveryAction, "recoveryAction must not be null."); this.producerOffsetsPollIntervalMs = producerOffsetsPollIntervalMs; this.maxPollTimeoutMs = maxPollTimeoutMs; this.offsetReporterRegistryId = offsetReporterRegistryId; - // Call setup internally - this is allowed because we're inside the operator class this.setup( parameters.getContainingTask(), parameters.getStreamConfig(), @@ -246,61 +231,111 @@ public UndoRecoveryOperator( public void initializeState(StateInitializationContext context) throws Exception { super.initializeState(context); - // Use RuntimeContextAdapter for Flink version compatibility - // (getTaskInfo() was added in Flink 1.19, direct methods deprecated in Flink 2.x) StreamingRuntimeContext runtimeContext = (StreamingRuntimeContext) getRuntimeContext(); parallelism = RuntimeContextAdapter.getNumberOfParallelSubtasks(runtimeContext); subtaskIndex = RuntimeContextAdapter.getIndexOfThisSubtask(runtimeContext); producerOffsetsDeleted = false; restoredFromCheckpoint = context.isRestored(); - // Resolve producerId: use configured value or default to Flink job ID - resolvedProducerId = configuredProducerId; - if (resolvedProducerId == null) { - resolvedProducerId = RuntimeContextAdapter.getJobId(getRuntimeContext()).toString(); - LOG.info("Using Flink job ID as producerId: {}", resolvedProducerId); - } - LOG.info( - "Initializing UndoRecoveryOperator for table {} (subtask {}/{}, producerId={})", + "Initializing UndoRecoveryOperator for table {} with action {} (subtask {}/{})", tablePath, + recoveryAction, subtaskIndex, - parallelism, - resolvedProducerId); + parallelism); + + recoveryActionState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + RECOVERY_ACTION_STATE_NAME, StringSerializer.INSTANCE)); + validateRecoveryActionTransition(context.isRestored()); - // Step 1: Get Union List State - // Union List State ensures each subtask receives a complete copy of all states during - // recovery undoStateList = context.getOperatorStateStore() .getUnionListState( new ListStateDescriptor<>( UNDO_RECOVERY_STATE_NAME, new WriterStateSerializer())); - // Step 2: Convert Union List State to Collection for RecoveryOffsetManager - // Note: context.isRestored() == false means fresh start (no checkpoint exists) - // context.isRestored() == true means restored from checkpoint + if (recoveryAction == RecoveryAction.NO_OP) { + initializeNoOpState(); + LOG.info("UndoRecoveryOperator initialized with NO_OP for subtask {}", subtaskIndex); + return; + } + Collection recoveredState = null; if (context.isRestored()) { recoveredState = new ArrayList<>(); for (WriterState state : undoStateList.get()) { recoveredState.add(state); } - LOG.debug( - "Restored {} WriterState objects from Union List State for subtask {}", - recoveredState.size(), - subtaskIndex); - // Log detailed state content for debugging recovery issues - for (WriterState state : recoveredState) { - LOG.debug( - "Subtask {} restored WriterState: bucketOffsets={}", - subtaskIndex, - state.getBucketOffsets()); + } + initializeUndoRecovery(recoveredState); + } + + private void validateRecoveryActionTransition(boolean restored) throws Exception { + if (!restored) { + return; + } + + RecoveryAction checkpointAction = + resolveCheckpointRecoveryAction(recoveryActionState.get()); + if (checkpointAction != null) { + validateRecoveryActionTransition(checkpointAction, recoveryAction); + } + } + + @Nullable + private static RecoveryAction resolveCheckpointRecoveryAction(Iterable actionNames) { + RecoveryAction checkpointAction = null; + for (String actionName : actionNames) { + RecoveryAction action; + try { + action = RecoveryAction.valueOf(actionName); + } catch (IllegalArgumentException e) { + throw new IllegalStateException( + "Unknown recovery action marker in checkpoint: " + actionName, e); } + if (checkpointAction != null && checkpointAction != action) { + throw new IllegalStateException( + "Found conflicting recovery action markers in checkpoint: " + + checkpointAction + + " and " + + action + + '.'); + } + checkpointAction = action; } + return checkpointAction; + } + + private static void validateRecoveryActionTransition( + RecoveryAction checkpointAction, RecoveryAction currentAction) { + if (checkpointAction == RecoveryAction.NO_OP && currentAction == RecoveryAction.UNDO) { + throw new IllegalStateException( + "Cannot restore recovery action UNDO from a checkpoint created with NO_OP. " + + "NO_OP checkpoints do not retain offsets required by undo recovery."); + } + } + + private void initializeNoOpState() throws Exception { + initializeBucketOffsets(Collections.emptyMap()); + if (subtaskIndex != 0 || configuredProducerId == null) { + return; + } + + resolvedProducerId = configuredProducerId; + LOG.info("Task0 deleting stale producer offsets for producerId {}", resolvedProducerId); + try (Connection cleanupConnection = ConnectionFactory.createConnection(flussConfig)) { + cleanupConnection.getAdmin().deleteProducerOffsets(resolvedProducerId).get(); + } + LOG.info("Task0 deleted stale producer offsets for producerId {}", resolvedProducerId); + } + + private void initializeUndoRecovery(@Nullable Collection recoveredState) + throws Exception { + resolvedProducerId = resolveProducerId(); - // Step 3: Use RecoveryOffsetManager to determine recovery strategy - // This handles both checkpoint recovery and producer offset recovery initializeFlussConnection(); if (table == null) { table = connection.getTable(tablePath); @@ -316,41 +351,45 @@ public void initializeState(StateInitializationContext context) throws Exception maxPollTimeoutMs, tablePath, table.getTableInfo()); - RecoveryOffsetManager.RecoveryDecision decision = offsetManager.determineRecoveryStrategy(recoveredState); - LOG.info("Recovery decision for subtask {}: {}", subtaskIndex, decision); + applyRecoveryDecision(decision); + LOG.info( + "UndoRecoveryOperator initialized for subtask {} with {} bucket offsets", + subtaskIndex, + bucketOffsets.size()); + } + + private String resolveProducerId() { + if (configuredProducerId != null) { + return configuredProducerId; + } + String producerId = RuntimeContextAdapter.getJobId(getRuntimeContext()).toString(); + LOG.info("Using Flink job ID as producerId: {}", producerId); + return producerId; + } - // Step 4: Execute undo recovery if needed + private void applyRecoveryDecision(RecoveryOffsetManager.RecoveryDecision decision) + throws Exception { if (decision.needsUndoRecovery()) { Map undoOffsets = decision.getUndoOffsets(); LOG.info( "Executing undo recovery for subtask {}: {} buckets", subtaskIndex, undoOffsets.size()); - LOG.debug("Subtask {} undoOffsets details: {}", subtaskIndex, undoOffsets); - performUndoRecovery(undoOffsets); - // Initialize bucket offsets with recovery offsets (checkpoint offsets) Map recoveryOffsets = decision.getRecoveryOffsets(); LOG.info( "Subtask {} initializing bucketOffsets from recovery: {} buckets", subtaskIndex, recoveryOffsets.size()); - LOG.debug("Subtask {} recovery offsets details: {}", subtaskIndex, recoveryOffsets); initializeBucketOffsets(recoveryOffsets); } else { LOG.info("No undo recovery needed for subtask {}", subtaskIndex); - // Initialize empty bucket offsets - initializeBucketOffsets(new HashMap<>()); + initializeBucketOffsets(Collections.emptyMap()); } - - LOG.info( - "UndoRecoveryOperator initialized for subtask {} with {} bucket offsets", - subtaskIndex, - bucketOffsets.size()); } /** @@ -403,14 +442,12 @@ private void performUndoRecovery(Map undoOffsets) thro /** * Snapshots the current state during checkpoint. * - *

This method is called by Flink during checkpoint processing. It clears the existing state - * list and adds a new {@link WriterState} with the current bucket offsets if the map is not - * empty. + *

The action marker is stored once in Union List State. UNDO also stores the current bucket + * offsets. NO_OP clears any restored undo state and writes no replacement. * - *

Note: Producer offset cleanup is NOT done here. It is done in {@link - * #notifyCheckpointComplete(long)} to ensure the checkpoint is fully committed before deleting - * the producer offsets. This prevents data loss in case of failure between snapshotState and - * checkpoint completion. + *

For UNDO, producer offset cleanup is not done here. It is done after checkpoint completion + * to prevent data loss between snapshot and completion. NO_OP cleans up an explicitly + * configured producer ID during initialization. * * @param context the state snapshot context containing checkpoint information * @throws Exception if snapshotting fails @@ -419,9 +456,22 @@ private void performUndoRecovery(Map undoOffsets) thro public void snapshotState(StateSnapshotContext context) throws Exception { super.snapshotState(context); + recoveryActionState.clear(); + if (subtaskIndex == 0) { + recoveryActionState.add(recoveryAction.name()); + } + // Clear existing state undoStateList.clear(); + if (recoveryAction == RecoveryAction.NO_OP) { + LOG.debug( + "Subtask {} cleared undo state at NO_OP checkpoint {}", + subtaskIndex, + context.getCheckpointId()); + return; + } + // Add new state if bucket offsets is not empty if (bucketOffsets != null) { if (!bucketOffsets.isEmpty()) { @@ -447,7 +497,7 @@ public void snapshotState(StateSnapshotContext context) throws Exception { } /** - * Called when a checkpoint is completed successfully. + * Called when an UNDO checkpoint is completed successfully. * *

This method triggers producer offset cleanup after the first successful checkpoint, but * ONLY when the operator was restored from a checkpoint. This is critical for the producer @@ -477,7 +527,10 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { // Only delete producer offsets if we were restored from a checkpoint. // If starting fresh, keep producer offsets for potential recovery on failure. - if (restoredFromCheckpoint && bucketOffsets != null && !bucketOffsets.isEmpty()) { + if (recoveryAction == RecoveryAction.UNDO + && restoredFromCheckpoint + && bucketOffsets != null + && !bucketOffsets.isEmpty()) { deleteProducerOffsetsIfNeeded(); } } @@ -489,7 +542,7 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { * recovery scenarios. Only Task0 performs the deletion to avoid concurrent cleanup attempts. */ private void deleteProducerOffsetsIfNeeded() { - if (producerOffsetsDeleted) { + if (recoveryAction == RecoveryAction.NO_OP || producerOffsetsDeleted) { return; } producerOffsetsDeleted = true; @@ -534,9 +587,8 @@ public void processElement(StreamRecord element) throws Exception { /** * Called when the input is exhausted (for bounded streams). * - *

Cleans up producer offsets in ZooKeeper to prevent stale entries from lingering. This is - * important for bounded jobs where {@link #notifyCheckpointComplete} may never be called (e.g., - * checkpointing not enabled, or job finishes before the first checkpoint). + *

For UNDO, cleans up producer offsets to prevent stale entries from lingering. This is + * important for bounded jobs where {@link #notifyCheckpointComplete} may never be called. * *

The cleanup is idempotent — {@link #deleteProducerOffsetsIfNeeded()} uses the {@code * producerOffsetsDeleted} flag and Task0-only guard, so it's safe to call from both here and @@ -546,7 +598,9 @@ public void processElement(StreamRecord element) throws Exception { */ @Override public void endInput() throws Exception { - deleteProducerOffsetsIfNeeded(); + if (recoveryAction == RecoveryAction.UNDO) { + deleteProducerOffsetsIfNeeded(); + } } // ==================== ProducerOffsetReporter Methods ==================== @@ -554,8 +608,9 @@ public void endInput() throws Exception { /** * Reports a written offset for a bucket. * - *

This method is called from async write callbacks on multiple threads. Thread-safety is - * provided by the ConcurrentHashMap's atomic {@code merge()} operation. + *

NO_OP ignores reports defensively. For UNDO, this method is called from async write + * callbacks on multiple threads. Thread-safety is provided by the ConcurrentHashMap's atomic + * {@code merge()} operation. * *

The method updates the offset only if the new offset is greater than the existing one, * ensuring monotonically increasing offsets for each bucket. @@ -565,6 +620,9 @@ public void endInput() throws Exception { */ @Override public void reportOffset(TableBucket bucket, long offset) { + if (recoveryAction == RecoveryAction.NO_OP) { + return; + } if (bucketOffsets != null) { bucketOffsets.merge(bucket, offset, Math::max); if (LOG.isTraceEnabled()) { @@ -660,6 +718,7 @@ public boolean isPartitioned() { return isPartitioned; } + @Nullable public String getProducerId() { return resolvedProducerId != null ? resolvedProducerId : configuredProducerId; } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorFactory.java index 5945429a46..899f437810 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorFactory.java @@ -49,11 +49,9 @@ * chaining. This allows the UndoRecoveryOperator to be chained with downstream operators (like the * SinkWriter) for better performance by reducing serialization overhead and network communication. * - *

ProducerOffsetReporter: The factory creates a shared {@link - * ProducerOffsetReporterHolder} that acts as a bridge between the factory and the operator. The - * holder is passed to the downstream SinkWriter via {@link #getProducerOffsetReporter()}, and when - * the operator is created at runtime, it registers itself with the holder. This enables the - * SinkWriter to report written offsets back to the operator for state tracking. + *

ProducerOffsetReporter: For UNDO, the factory's shared {@link + * ProducerOffsetReporterHolder} bridges the downstream SinkWriter and the operator. NO_OP keeps the + * same operator topology but does not attach or register this callback. * * @param The type of input elements * @see UndoRecoveryOperator @@ -89,10 +87,13 @@ public class UndoRecoveryOperatorFactory extends AbstractStreamOperatorFacto * The producer ID used for producer offset snapshot management. * *

This is used by {@link RecoveryOffsetManager} to register and retrieve producer offsets - * for pre-checkpoint failure recovery. If null, the operator will use the Flink job ID. + * for pre-checkpoint failure recovery. If null, UNDO uses the Flink job ID while NO_OP performs + * no producer-offset operation. */ @Nullable private final String producerId; + private final RecoveryAction recoveryAction; + /** The polling interval in milliseconds for producer offsets synchronization. */ private final long producerOffsetsPollIntervalMs; @@ -104,24 +105,24 @@ public class UndoRecoveryOperatorFactory extends AbstractStreamOperatorFacto /** * The shared ProducerOffsetReporter holder. * - *

This holder is created in the constructor and passed to both the downstream SinkWriter - * (via {@link #getProducerOffsetReporter()}) and the operator (when created at runtime). The - * holder delegates offset reports to the actual operator once it's registered. + *

For UNDO, this holder is passed to the downstream SinkWriter via {@link + * #getProducerOffsetReporter()} and delegates offset reports to the operator once registered. */ private final ProducerOffsetReporterHolder offsetReporterHolder; // ==================== Constructors ==================== /** - * Creates a new UndoRecoveryOperatorFactory with default producer offset poll interval. + * Creates a factory with an explicit failover correction action. * - * @param tablePath the table path for the Fluss table - * @param flussConfig the Fluss configuration - * @param tableRowType the row type of the table - * @param targetColumnIndexes target column indexes for partial update (null for full row) - * @param numBuckets the number of buckets in the table + * @param tablePath the target table + * @param flussConfig client configuration + * @param tableRowType target row type + * @param targetColumnIndexes partial-update targets, or null for full update + * @param numBuckets table bucket count * @param isPartitioned whether the table is partitioned - * @param producerId the producer ID for producer offset management (null to use Flink job ID) + * @param producerId producer ID, or null to use the Flink job ID for UNDO only + * @param recoveryAction failover correction action */ public UndoRecoveryOperatorFactory( TablePath tablePath, @@ -130,7 +131,8 @@ public UndoRecoveryOperatorFactory( @Nullable int[] targetColumnIndexes, int numBuckets, boolean isPartitioned, - @Nullable String producerId) { + @Nullable String producerId, + RecoveryAction recoveryAction) { this( tablePath, flussConfig, @@ -139,27 +141,12 @@ public UndoRecoveryOperatorFactory( numBuckets, isPartitioned, producerId, + recoveryAction, RecoveryOffsetManager.DEFAULT_PRODUCER_OFFSETS_POLL_INTERVAL_MS, RecoveryOffsetManager.DEFAULT_MAX_POLL_TIMEOUT_MS); } - /** - * Creates a new UndoRecoveryOperatorFactory. - * - *

The factory is configured with {@link ChainingStrategy#ALWAYS} to enable operator chaining - * with downstream operators for better performance. - * - * @param tablePath the table path for the Fluss table - * @param flussConfig the Fluss configuration - * @param tableRowType the row type of the table - * @param targetColumnIndexes target column indexes for partial update (null for full row) - * @param numBuckets the number of buckets in the table - * @param isPartitioned whether the table is partitioned - * @param producerId the producer ID for producer offset management (null to use Flink job ID) - * @param producerOffsetsPollIntervalMs the polling interval for producer offsets - * @param maxPollTimeoutMs the maximum total time to poll for producer offsets - */ - public UndoRecoveryOperatorFactory( + UndoRecoveryOperatorFactory( TablePath tablePath, Configuration flussConfig, RowType tableRowType, @@ -167,6 +154,7 @@ public UndoRecoveryOperatorFactory( int numBuckets, boolean isPartitioned, @Nullable String producerId, + RecoveryAction recoveryAction, long producerOffsetsPollIntervalMs, long maxPollTimeoutMs) { this.tablePath = tablePath; @@ -176,6 +164,7 @@ public UndoRecoveryOperatorFactory( this.numBuckets = numBuckets; this.isPartitioned = isPartitioned; this.producerId = producerId; + this.recoveryAction = recoveryAction; this.producerOffsetsPollIntervalMs = producerOffsetsPollIntervalMs; this.maxPollTimeoutMs = maxPollTimeoutMs; @@ -193,8 +182,8 @@ public UndoRecoveryOperatorFactory( * Creates a new {@link UndoRecoveryOperator} instance. * *

This method is called by Flink's runtime to create the operator instance. The created - * operator is registered with the {@link ProducerOffsetReporterHolder} so that offset reports - * from the downstream SinkWriter are forwarded to the operator. + * operator is registered with the {@link ProducerOffsetReporterHolder} for UNDO so that offset + * reports from the downstream SinkWriter are forwarded to the operator. * * @param parameters the stream operator parameters from Flink runtime * @param the type of the stream operator @@ -213,12 +202,14 @@ public > T createStreamOperator( numBuckets, isPartitioned, producerId, + recoveryAction, producerOffsetsPollIntervalMs, maxPollTimeoutMs, offsetReporterHolder.getHolderId()); - // Register the operator with the static registry so offset reports are forwarded - registerDelegate(offsetReporterHolder.getHolderId(), operator); + if (recoveryAction == RecoveryAction.UNDO) { + registerDelegate(offsetReporterHolder.getHolderId(), operator); + } @SuppressWarnings("unchecked") final T castedOperator = (T) operator; @@ -241,7 +232,7 @@ public Class getStreamOperatorClass(ClassLoader classL // ==================== Getters ==================== /** - * Returns the ProducerOffsetReporter that can be passed to the downstream SinkWriter. + * Returns the ProducerOffsetReporter for an UNDO sink path. * * @return the ProducerOffsetReporter holder */ @@ -279,6 +270,15 @@ public String getProducerId() { return producerId; } + /** + * Returns the failover correction action. + * + * @return NO_OP or UNDO + */ + public RecoveryAction getRecoveryAction() { + return recoveryAction; + } + public long getProducerOffsetsPollIntervalMs() { return producerOffsetsPollIntervalMs; } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/UpsertSinkWriter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/UpsertSinkWriter.java index 73ac142f04..5d6c2c7c81 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/UpsertSinkWriter.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/UpsertSinkWriter.java @@ -49,7 +49,8 @@ public class UpsertSinkWriter extends FlinkSinkWriter { *

When provided (non-null), the writer will report offsets after each successful write * operation. This is used when the UndoRecoveryOperator manages state for aggregation tables. * - *

When null, offset reporting is skipped (for non-aggregation tables). + *

When null, offset reporting is skipped for non-aggregation tables and for aggregation + * tables using {@code RecoveryAction.NO_OP}. */ @Nullable private final ProducerOffsetReporter offsetReporter; diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/AggregationRecoveryDeciderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/AggregationRecoveryDeciderTest.java new file mode 100644 index 0000000000..b5acbe69f8 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/AggregationRecoveryDeciderTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink; + +import org.apache.fluss.flink.sink.undo.RecoveryAction; +import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; + +import static org.apache.fluss.metadata.AggFunctions.MAX; +import static org.apache.fluss.metadata.AggFunctions.SUM; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link AggregationRecoveryDecider}. */ +class AggregationRecoveryDeciderTest { + + private static final Schema MIXED_SCHEMA = + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("max_value", DataTypes.BIGINT(), MAX()) + .column("sum_value", DataTypes.BIGINT(), SUM()) + .column("implicit_last", DataTypes.STRING()) + .primaryKey("id") + .build(); + private static final Schema SAFE_SCHEMA = + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("max_value", DataTypes.BIGINT(), MAX()) + .column("implicit_last", DataTypes.STRING()) + .primaryKey("id") + .build(); + + @Test + void testFullAndPartialTargets() { + assertThat(decide(SAFE_SCHEMA, null, DeleteBehavior.ALLOW, false, true)) + .isEqualTo(RecoveryAction.NO_OP); + assertThat(decide(MIXED_SCHEMA, null, DeleteBehavior.ALLOW, false, true)) + .isEqualTo(RecoveryAction.UNDO); + assertThat(decide(MIXED_SCHEMA, new int[] {0, 1, 3}, DeleteBehavior.ALLOW, false, true)) + .isEqualTo(RecoveryAction.NO_OP); + assertThat(decide(MIXED_SCHEMA, new int[] {0, 2}, DeleteBehavior.ALLOW, false, true)) + .isEqualTo(RecoveryAction.UNDO); + } + + @Test + void testPrimaryKeyAndImplicitLastValueAreSafe() { + assertThat(decide(SAFE_SCHEMA, new int[] {0}, DeleteBehavior.ALLOW, false, true)) + .isEqualTo(RecoveryAction.NO_OP); + assertThat(decide(SAFE_SCHEMA, new int[] {0, 2}, DeleteBehavior.ALLOW, false, true)) + .isEqualTo(RecoveryAction.NO_OP); + } + + @Test + void testInputNotKnownUpsertOnlyUsesConservativeDeletePolicy() { + assertThat(decide(SAFE_SCHEMA, null, DeleteBehavior.ALLOW, false, false)) + .isEqualTo(RecoveryAction.UNDO); + assertThat(decide(SAFE_SCHEMA, null, DeleteBehavior.IGNORE, false, false)) + .isEqualTo(RecoveryAction.NO_OP); + assertThat(decide(SAFE_SCHEMA, null, DeleteBehavior.DISABLE, false, false)) + .isEqualTo(RecoveryAction.NO_OP); + assertThat(decide(SAFE_SCHEMA, null, DeleteBehavior.ALLOW, true, false)) + .isEqualTo(RecoveryAction.NO_OP); + } + + @Test + void testInvalidTargetsStillFailFast() { + assertThatThrownBy( + () -> + decide( + SAFE_SCHEMA, + new int[] {-1}, + DeleteBehavior.ALLOW, + false, + false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid target column index -1"); + assertThatThrownBy( + () -> + decide( + SAFE_SCHEMA, + new int[] {3}, + DeleteBehavior.ALLOW, + false, + false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid target column index 3"); + } + + private static RecoveryAction decide( + Schema schema, + int[] targets, + DeleteBehavior deleteBehavior, + boolean ignoreDelete, + boolean inputKnownUpsertOnly) { + return AggregationRecoveryDecider.decide( + schema, targets, deleteBehavior, ignoreDelete, inputKnownUpsertOnly); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkSinkTopologyTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkSinkTopologyTest.java new file mode 100644 index 0000000000..662c118cff --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkSinkTopologyTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink; + +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.row.RowWithOp; +import org.apache.fluss.flink.sink.serializer.FlussSerializationSchema; +import org.apache.fluss.flink.sink.shuffle.DistributionMode; +import org.apache.fluss.flink.sink.undo.RecoveryAction; +import org.apache.fluss.flink.sink.undo.UndoRecoveryOperatorFactory; +import org.apache.fluss.metadata.TablePath; + +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.operators.ChainingStrategy; +import org.apache.flink.streaming.api.transformations.OneInputTransformation; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.logical.RowType; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests the pre-write topology built by {@link FlinkSink.UpsertSinkWriterBuilder}. */ +class FlinkSinkTopologyTest { + + private static final TablePath TABLE_PATH = TablePath.of("db", "table"); + private static final RowType ROW_TYPE = + (RowType) DataTypes.ROW(DataTypes.FIELD("id", DataTypes.BIGINT())).getLogicalType(); + private static final FlussSerializationSchema SERIALIZATION_SCHEMA = + new FlussSerializationSchema() { + private static final long serialVersionUID = 1L; + + @Override + public void open(InitializationContext context) {} + + @Override + public RowWithOp serialize(Integer value) { + return null; + } + }; + + @Test + void testNoOpKeepsUndoRecoveryTransformIdentity() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(2); + DataStream input = env.fromElements(1); + + FlinkSink.UpsertSinkWriterBuilder builder = + createBuilder(true, RecoveryAction.NO_OP); + DataStream transformed = builder.addPreWriteTopology(input); + + @SuppressWarnings("unchecked") + OneInputTransformation transformation = + (OneInputTransformation) transformed.getTransformation(); + assertThat(transformation.getName()).isEqualTo("UndoRecovery(db.table)"); + assertThat(transformation.getParallelism()).isEqualTo(input.getParallelism()); + assertThat(transformation.getOperatorFactory()) + .isInstanceOf(UndoRecoveryOperatorFactory.class); + + UndoRecoveryOperatorFactory factory = + (UndoRecoveryOperatorFactory) transformation.getOperatorFactory(); + assertThat(factory.getRecoveryAction()).isEqualTo(RecoveryAction.NO_OP); + assertThat(factory.getChainingStrategy()).isEqualTo(ChainingStrategy.ALWAYS); + assertThat(builder).extracting("offsetReporter").isNull(); + } + + @Test + void testUndoInstallsOffsetReporter() { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + DataStream input = env.fromElements(1); + + FlinkSink.UpsertSinkWriterBuilder builder = + createBuilder(true, RecoveryAction.UNDO); + DataStream transformed = builder.addPreWriteTopology(input); + OneInputTransformation transformation = + (OneInputTransformation) transformed.getTransformation(); + UndoRecoveryOperatorFactory factory = + (UndoRecoveryOperatorFactory) transformation.getOperatorFactory(); + assertThat(factory.getRecoveryAction()).isEqualTo(RecoveryAction.UNDO); + assertThat(builder).extracting("offsetReporter").isNotNull(); + } + + private static FlinkSink.UpsertSinkWriterBuilder createBuilder( + boolean enableUndoRecovery, RecoveryAction action) { + return new FlinkSink.UpsertSinkWriterBuilder<>( + TABLE_PATH, + new Configuration(), + ROW_TYPE, + null, + 1, + Collections.singletonList("id"), + Collections.emptyList(), + null, + DistributionMode.NONE, + SERIALIZATION_SCHEMA, + enableUndoRecovery, + action, + null); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkTest.java new file mode 100644 index 0000000000..7c3a902ce2 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink; + +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.sink.shuffle.DistributionMode; +import org.apache.fluss.flink.sink.undo.RecoveryAction; +import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.MergeEngineType; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.types.DataTypes; + +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.RowKind; +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.apache.fluss.metadata.AggFunctions.MAX; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests recovery-action plumbing in {@link FlinkTableSink}. */ +class FlinkTableSinkTest { + + private static final Schema AGG_SCHEMA = + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("max_value", DataTypes.BIGINT(), MAX()) + .primaryKey("id") + .build(); + private static final RowType FLINK_ROW_TYPE = + (RowType) + org.apache.flink.table.api.DataTypes.ROW( + org.apache.flink.table.api.DataTypes.FIELD( + "id", org.apache.flink.table.api.DataTypes.BIGINT()), + org.apache.flink.table.api.DataTypes.FIELD( + "max_value", + org.apache.flink.table.api.DataTypes.BIGINT())) + .getLogicalType(); + + @Test + void testInsertOnlyRecoveryProofSurvivesCopy() { + FlinkTableSink sink = createSink(AGG_SCHEMA, false, DeleteBehavior.ALLOW); + assertThat(sink.getChangelogMode(ChangelogMode.insertOnly())) + .isEqualTo(ChangelogMode.insertOnly()); + + assertThat(sink.getRecoveryAction(new int[] {0, 1})).isEqualTo(RecoveryAction.NO_OP); + + FlinkTableSink copied = (FlinkTableSink) sink.copy(); + assertThat(copied.getRecoveryAction(new int[] {0, 1})).isEqualTo(RecoveryAction.NO_OP); + } + + @Test + void testEveryObservedChangelogModeMustProveUpsertOnlyInput() { + FlinkTableSink insertThenAll = createSink(AGG_SCHEMA, false, DeleteBehavior.ALLOW); + insertThenAll.getChangelogMode(ChangelogMode.insertOnly()); + insertThenAll.getChangelogMode(ChangelogMode.all()); + + assertThat(insertThenAll.getRecoveryAction(new int[] {0, 1})) + .isEqualTo(RecoveryAction.UNDO); + FlinkTableSink copied = (FlinkTableSink) insertThenAll.copy(); + copied.getChangelogMode(ChangelogMode.insertOnly()); + assertThat(copied.getRecoveryAction(new int[] {0, 1})).isEqualTo(RecoveryAction.UNDO); + + FlinkTableSink allThenInsert = createSink(AGG_SCHEMA, false, DeleteBehavior.ALLOW); + allThenInsert.getChangelogMode(ChangelogMode.all()); + allThenInsert.getChangelogMode(ChangelogMode.insertOnly()); + + assertThat(allThenInsert.getRecoveryAction(new int[] {0, 1})) + .isEqualTo(RecoveryAction.UNDO); + } + + @Test + void testAcceptedAndIgnoredDeletesSelectRecoveryAction() { + FlinkTableSink deleting = createSink(AGG_SCHEMA, false, DeleteBehavior.ALLOW); + ChangelogMode acceptedMode = deleting.getChangelogMode(ChangelogMode.all()); + assertThat(acceptedMode.getContainedKinds()) + .containsExactlyInAnyOrder(RowKind.INSERT, RowKind.UPDATE_AFTER, RowKind.DELETE); + assertThat(deleting.getRecoveryAction(new int[] {0, 1})).isEqualTo(RecoveryAction.UNDO); + assertThat(((FlinkTableSink) deleting.copy()).getRecoveryAction(new int[] {0, 1})) + .isEqualTo(RecoveryAction.UNDO); + + FlinkTableSink ignoring = createSink(AGG_SCHEMA, true, DeleteBehavior.ALLOW); + ignoring.getChangelogMode(ChangelogMode.all()); + assertThat(ignoring.getRecoveryAction(new int[] {0, 1})).isEqualTo(RecoveryAction.NO_OP); + } + + @Test + void testLegacyConstructorConservativelyUsesUndoAfterCopy() { + FlinkTableSink sink = createLegacySink(); + sink.getChangelogMode(ChangelogMode.insertOnly()); + + assertThat(sink.getRecoveryAction(new int[] {0, 1})).isEqualTo(RecoveryAction.UNDO); + assertThat(((FlinkTableSink) sink.copy()).getRecoveryAction(new int[] {0, 1})) + .isEqualTo(RecoveryAction.UNDO); + } + + private static FlinkTableSink createSink( + Schema schema, boolean ignoreDelete, DeleteBehavior deleteBehavior) { + return new FlinkTableSink( + TablePath.of("db", "mixed"), + new Configuration(), + FLINK_ROW_TYPE, + schema, + new int[] {0}, + Collections.emptyList(), + true, + MergeEngineType.AGGREGATION, + null, + ignoreDelete, + deleteBehavior, + 1, + Collections.singletonList("id"), + DistributionMode.NONE, + null); + } + + private static FlinkTableSink createLegacySink() { + return new FlinkTableSink( + TablePath.of("db", "legacy"), + new Configuration(), + FLINK_ROW_TYPE, + new int[] {0}, + Collections.emptyList(), + true, + MergeEngineType.AGGREGATION, + null, + false, + DeleteBehavior.ALLOW, + 1, + Collections.singletonList("id"), + DistributionMode.NONE, + null); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlussSinkBuilderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlussSinkBuilderTest.java index 19a724251e..d910379f1f 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlussSinkBuilderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlussSinkBuilderTest.java @@ -19,13 +19,18 @@ import org.apache.fluss.flink.sink.serializer.OrderSerializationSchema; import org.apache.fluss.flink.sink.shuffle.DistributionMode; +import org.apache.fluss.flink.sink.undo.RecoveryAction; import org.apache.fluss.flink.source.testutils.Order; +import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.TableInfo; +import org.apache.fluss.metadata.TablePath; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.lang.reflect.Field; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -226,6 +231,57 @@ void testComputeTargetColumnIndexesUnknownColumnThrows() { .hasMessageContaining("not found in table schema"); } + @Test + void testDataStreamRecoveryActionUsesDeletePolicyAndEffectiveTargets() { + assertThat( + FlussSinkBuilder.determineRecoveryAction( + createAggregationTableInfo(DeleteBehavior.ALLOW), new int[] {0, 1})) + .isEqualTo(RecoveryAction.UNDO); + + TableInfo ignoringDeletes = createAggregationTableInfo(DeleteBehavior.IGNORE); + assertThat(FlussSinkBuilder.determineRecoveryAction(ignoringDeletes, new int[] {0, 1})) + .isEqualTo(RecoveryAction.NO_OP); + assertThat(FlussSinkBuilder.determineRecoveryAction(ignoringDeletes, new int[] {0, 2})) + .isEqualTo(RecoveryAction.UNDO); + } + + private static TableInfo createAggregationTableInfo(DeleteBehavior deleteBehavior) { + org.apache.fluss.metadata.Schema schema = + org.apache.fluss.metadata.Schema.newBuilder() + .column("id", org.apache.fluss.types.DataTypes.BIGINT()) + .column( + "max_value", + org.apache.fluss.types.DataTypes.BIGINT(), + org.apache.fluss.metadata.AggFunctions.MAX()) + .column( + "sum_value", + org.apache.fluss.types.DataTypes.BIGINT(), + org.apache.fluss.metadata.AggFunctions.SUM()) + .primaryKey("id") + .build(); + org.apache.fluss.config.Configuration properties = + new org.apache.fluss.config.Configuration(); + properties.set( + org.apache.fluss.config.ConfigOptions.TABLE_MERGE_ENGINE, + org.apache.fluss.metadata.MergeEngineType.AGGREGATION); + properties.set(org.apache.fluss.config.ConfigOptions.TABLE_DELETE_BEHAVIOR, deleteBehavior); + + return new TableInfo( + TablePath.of("db", "mixed"), + 1L, + 1, + schema, + Collections.singletonList("id"), + Collections.emptyList(), + 1, + properties, + new org.apache.fluss.config.Configuration(), + null, + null, + 0L, + 0L); + } + // Helper method to get private field values using reflection @SuppressWarnings("unchecked") private T getFieldValue(Object object, String fieldName) throws Exception { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/UndoRecoveryITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/UndoRecoveryITCase.java index df023ed7a7..78c4d2ca6f 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/UndoRecoveryITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/UndoRecoveryITCase.java @@ -20,13 +20,17 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.admin.ProducerOffsetsResult; +import org.apache.fluss.client.admin.RegisterResult; import org.apache.fluss.client.lookup.Lookuper; import org.apache.fluss.client.table.Table; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.sink.serializer.RowDataSerializationSchema; import org.apache.fluss.flink.sink.testutils.CountingSource; import org.apache.fluss.flink.sink.testutils.FailingCountingSource; +import org.apache.fluss.flink.sink.undo.RecoveryAction; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.row.InternalRow; import org.apache.fluss.server.testutils.FlussClusterExtension; @@ -40,6 +44,8 @@ import org.apache.flink.configuration.StateBackendOptions; import org.apache.flink.core.execution.JobClient; import org.apache.flink.core.execution.SavepointFormatType; +import org.apache.flink.runtime.checkpoint.CheckpointException; +import org.apache.flink.runtime.checkpoint.CheckpointFailureReason; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -62,8 +68,15 @@ import java.io.File; import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import static org.apache.flink.test.util.TestUtils.waitUntilAllTasksAreRunning; import static org.apache.fluss.flink.FlinkConnectorOptions.BOOTSTRAP_SERVERS; import static org.apache.fluss.testutils.DataTestUtils.row; import static org.apache.fluss.testutils.common.CommonTestUtils.retry; @@ -73,22 +86,12 @@ /** * Integration tests for Undo Recovery functionality using Aggregation Merge Engine. * - *

This test suite verifies undo recovery in two categories: + *

This test suite verifies aggregation failover recovery in two categories: * *

Checkpoint Failover Tests (use FailingCountingSource)

* - *

These tests simulate real job failures by injecting exceptions and relying on Flink's - * automatic checkpoint-based recovery. All failover tests use the default producerId (Flink Job ID) - * to match production usage: - * - *

    - *
  • Checkpoint Failover Recovery: {@link #testCheckpointFailoverRecovery()} - Tests - * recovery when failure occurs after checkpoint completion - *
  • Producer Offset Recovery: {@link #testProducerOffsetRecoveryWithFailover()} - Tests - * recovery when failure occurs before any checkpoint - *
  • Multiple Keys Recovery: {@link #testUndoRecoveryMultipleKeys()} - Tests recovery - * across multiple keys after checkpoint - *
+ *

These tests use explicit source gates to establish a committed prefix, dirty writes, an + * injected failure, and complete post-restart emission before checking the stable result. * *

Savepoint Rescale Tests (use CountingSource)

* @@ -104,8 +107,8 @@ * specific parallelism and cannot be used for rescaling. Savepoints provide the portable state * format needed for parallelism changes. * - *

Tests use bounded sources for precise record tracking and exact expected value calculations - * with {@code isEqualTo()} assertions. + *

Failover tests use deterministic source gates, while rescale tests use bounded sources. Both + * paths verify stable materialized values with exact assertions. */ abstract class UndoRecoveryITCase { @@ -174,265 +177,214 @@ protected void afterEach() throws Exception { // ==================== Test Methods ==================== - /** - * Tests checkpoint-based failover recovery. - * - *

This test verifies that undo recovery works correctly when a job fails and automatically - * recovers from a checkpoint. The test uses FailingCountingSource to inject a failure at a - * controlled point. - * - *

Pattern: - * - *

    - *
  1. Phase 1: Write 10 records, checkpoint completes - *
  2. Phase 2: Failure is triggered (simulating crash) - *
  3. Phase 3: Auto-recover from checkpoint, undo any uncommitted writes, write 3 new records - *
- * - *

Expected result: Final sum = (10 + 3) * VALUE_PER_RECORD = 130 (Any writes after the - * checkpoint are undone during recovery) - */ + /** Tests DataStream-builder UNDO recovery after a completed checkpoint. */ @Test void testCheckpointFailoverRecovery() throws Exception { String tableName = "undo_checkpoint_failover_" + System.currentTimeMillis(); TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + String producerId = "undo-datastream-" + System.nanoTime(); - initTableEnvironment(null, false).executeSql(createAggTableDDL(tableName)); - - // Configure: 10 records before failure, 3 records after recovery - int recordsBeforeFailure = 10; - int recordsAfterFailure = 3; - - // Start job with FailingCountingSource (default producerId = Flink Job ID) - JobClient job = - startFailoverJob( - tablePath, null, recordsBeforeFailure, recordsAfterFailure, 1, false); - - // Wait for checkpoint to complete before failure triggers - waitForCheckpoint(job.getJobID()); + initTableEnvironment(null, false) + .executeSql(createAggTableDDL(tableName, DEFAULT_BUCKET_NUM, "allow")); + FailingCountingSource source = + FailingCountingSource.coordinatedSingleKey( + failureMarkerDir, 1L, 10L, 1, 10L, 9, 10L, 3); + JobClient job = startCoordinatedFailoverDataStreamJob(tablePath, producerId, source); - // Wait for final sum after recovery - // Expected: (10 + 3) * VALUE_PER_RECORD = 130 - long expectedFinalSum = (recordsBeforeFailure + recordsAfterFailure) * VALUE_PER_RECORD; retry( DEFAULT_TIMEOUT, - () -> { - Long sum = lookupSum(tablePath, 1L); - assertThat(sum) - .as("Final sum after checkpoint failover recovery") - .isEqualTo(expectedFinalSum); - }); + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Committed SUM prefix") + .isEqualTo(10L)); + long initialCheckpointId = triggerCompletedCheckpointAfter(job.getJobID(), -1L); + assertRecoveryActionObservable(tablePath, producerId, RecoveryAction.UNDO); + long checkpointLogEndOffset = getLatestLogEndOffset(tablePath, 0); + + source.releaseDirtyEmission(); + waitForLogEndOffsetAtLeast(tablePath, 0, checkpointLogEndOffset + 9); + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Dirty SUM before failure") + .isEqualTo(100L)); + + triggerFailureAndWaitForRecovery(source); + long postRecoveryCheckpointId = + triggerCompletedCheckpointAfter(job.getJobID(), initialCheckpointId); + assertThat(postRecoveryCheckpointId).isGreaterThan(initialCheckpointId); + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Stable SUM after undo, replay, and recovery emission") + .isEqualTo(130L)); + assertProducerOffsetsAbsent(producerId); - // Cancel the job after verification job.cancel().get(); waitForJobTermination(job, DEFAULT_TIMEOUT); - - // Final verification - Long finalSum = lookupSum(tablePath, 1L); - LOG.info("Final sum: {}, expected: {}", finalSum, expectedFinalSum); - assertThat(finalSum) - .as( - "Final sum should equal (recordsBeforeFailure + recordsAfterFailure) * VALUE_PER_RECORD") - .isEqualTo(expectedFinalSum); + assertThat(lookupSum(tablePath, 1L)).isEqualTo(130L); } - /** Tests checkpoint-based failover recovery through the SQL/Table API sink path. */ + /** Tests DataStream-builder NO_OP recovery and one-time stale offset cleanup. */ @Test - void testCheckpointFailoverRecoverySQLJob() throws Exception { - String tableName = "undo_checkpoint_failover_sql_" + System.currentTimeMillis(); + void testDataStreamMaxFailoverUsesNoOpRecoveryAndCleansStaleOffsets() throws Exception { + String tableName = "no_op_datastream_max_" + System.currentTimeMillis(); TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + String producerId = "no-op-datastream-" + System.nanoTime(); - // Configure: 10 records before failure, 3 records after recovery - int recordsBeforeFailure = 10; - int recordsAfterFailure = 3; - - // Start job with FailingCountingSource (default producerId = Flink Job ID) - TableResult tableResult = - startFailoverSqlJob( - tablePath, - createAggTableDDL(tableName), - recordsBeforeFailure, - recordsAfterFailure, - 1, - false); - - JobClient job = - tableResult - .getJobClient() - .orElseThrow(() -> new RuntimeException("JobClient not available")); + initTableEnvironment(null, false) + .executeSql(createMaxAggTableDDL(tableName, producerId, "ignore")); + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + Map staleOffsets = + Collections.singletonMap(new TableBucket(tableId, 0), 0L); + assertThat(admin.registerProducerOffsets(producerId, staleOffsets).get()) + .isEqualTo(RegisterResult.CREATED); - // Wait for checkpoint to complete before failure triggers - waitForCheckpoint(job.getJobID()); + FailingCountingSource source = + FailingCountingSource.coordinatedSingleKey( + failureMarkerDir, 1L, 1L, 1, 10L, 1, 20L, 1); + JobClient job = startCoordinatedFailoverDataStreamJob(tablePath, producerId, source); - // Wait for final sum after recovery - // Expected: (10 + 3) * VALUE_PER_RECORD = 130 - long expectedFinalSum = (recordsBeforeFailure + recordsAfterFailure) * VALUE_PER_RECORD; retry( DEFAULT_TIMEOUT, - () -> { - Long sum = lookupSum(tablePath, 1L); - assertThat(sum) - .as("Final sum after checkpoint failover recovery") - .isEqualTo(expectedFinalSum); - }); + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Committed MAX prefix") + .isEqualTo(1L)); + assertProducerOffsetsAbsent(producerId); + long initialCheckpointId = triggerCompletedCheckpointAfter(job.getJobID(), -1L); + assertProducerOffsetsAbsent(producerId); + long checkpointLogEndOffset = getLatestLogEndOffset(tablePath, 0); + + source.releaseDirtyEmission(); + waitForLogEndOffsetAtLeast(tablePath, 0, checkpointLogEndOffset + 1); + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Dirty MAX before failure") + .isEqualTo(10L)); + assertProducerOffsetsAbsent(producerId); + + triggerFailureAndWaitForRecovery(source); + assertProducerOffsetsAbsent(producerId); + long postRecoveryCheckpointId = + triggerCompletedCheckpointAfter(job.getJobID(), initialCheckpointId); + assertThat(postRecoveryCheckpointId).isGreaterThan(initialCheckpointId); + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Recovery MAX value must not be dropped") + .isEqualTo(20L)); + assertProducerOffsetsAbsent(producerId); - // Cancel the job after verification job.cancel().get(); waitForJobTermination(job, DEFAULT_TIMEOUT); - - // Final verification - Long finalSum = lookupSum(tablePath, 1L); - LOG.info("Final sum: {}, expected: {}", finalSum, expectedFinalSum); - assertThat(finalSum) - .as( - "Final sum should equal (recordsBeforeFailure + recordsAfterFailure) * VALUE_PER_RECORD") - .isEqualTo(expectedFinalSum); + assertThat(lookupSum(tablePath, 1L)).isEqualTo(20L); + assertProducerOffsetsAbsent(producerId); + assertThat(admin.registerProducerOffsets(producerId, staleOffsets).get()) + .as("NO_OP must not leave an offset registration behind") + .isEqualTo(RegisterResult.CREATED); + admin.deleteProducerOffsets(producerId).get(); } - /** - * Tests undo recovery with multiple keys using checkpoint-based failover. - * - *

This test verifies that undo recovery works correctly across multiple keys when a job - * fails and automatically recovers from a checkpoint. The test uses FailingCountingSource in - * multi-key mode to emit records for keys 1, 2, 3. - * - *

Pattern: - * - *

    - *
  1. Phase 1: Write 10 records per key (30 total), checkpoint completes - *
  2. Phase 2: Failure is triggered (simulating crash) - *
  3. Phase 3: Auto-recover from checkpoint, undo any uncommitted writes, write 3 records per - * key - *
- * - *

Expected result: Each key's final sum = (10 + 3) * VALUE_PER_RECORD = 130 - */ @Test - void testUndoRecoveryMultipleKeys() throws Exception { - String tableName = "undo_multi_keys_" + System.currentTimeMillis(); + void testMixedAggregationSafePartialUpdateUsesNoOpRecovery() throws Exception { + String tableName = "no_op_mixed_partial_" + System.currentTimeMillis(); TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); - - initTableEnvironment(null, false).executeSql(createAggTableDDL(tableName)); - - // Configure: 10 records per key before failure, 3 records per key after recovery - int recordsBeforeFailurePerKey = 10; - int recordsAfterFailurePerKey = 3; - - // Start job with FailingCountingSource in multi-key mode (default producerId = Flink Job - // ID) - JobClient job = - startFailoverJob( + String producerId = "no-op-mixed-" + System.nanoTime(); + FailingCountingSource source = + FailingCountingSource.coordinatedSingleKey( + failureMarkerDir, 1L, 1L, 1, 10L, 1, 20L, 1); + TableResult result = + startFailoverSqlJob( tablePath, - null, - recordsBeforeFailurePerKey, - recordsAfterFailurePerKey, + createMixedAggTableDDL(tableName, producerId, "ignore"), 1, - true); // multiKey = true - - // Wait for checkpoint to complete before failure triggers - waitForCheckpoint(job.getJobID()); + Arrays.asList("id", "max_value"), + source, + TimeUnit.DAYS.toMillis(1)); + JobClient job = + result.getJobClient() + .orElseThrow(() -> new RuntimeException("JobClient not available")); - // Wait for final sums after recovery for all 3 keys - // Expected per key: (10 + 3) * VALUE_PER_RECORD = 130 - long expectedFinalSumPerKey = - (recordsBeforeFailurePerKey + recordsAfterFailurePerKey) * VALUE_PER_RECORD; retry( DEFAULT_TIMEOUT, - () -> { - Long[] sums = lookupSumsForKeys(tablePath, 1L, 2L, 3L); - for (int i = 0; i < 3; i++) { - assertThat(sums[i]) - .as( - "Key " - + (i + 1) - + " final sum after checkpoint failover recovery") - .isEqualTo(expectedFinalSumPerKey); - } - }); + () -> + assertThat(lookupLong(tablePath, 1L, 1)) + .as("Committed MAX prefix") + .isEqualTo(1L)); + long initialCheckpointId = triggerCompletedCheckpointAfter(job.getJobID(), -1L); + long checkpointLogEndOffset = getLatestLogEndOffset(tablePath, 0); + assertRecoveryActionObservable(tablePath, producerId, RecoveryAction.NO_OP); + + source.releaseDirtyEmission(); + waitForLogEndOffsetAtLeast(tablePath, 0, checkpointLogEndOffset + 1); + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(lookupLong(tablePath, 1L, 1)) + .as("Dirty MAX before failure") + .isEqualTo(10L)); + + triggerFailureAndWaitForRecovery(source); + assertThat(triggerCompletedCheckpointAfter(job.getJobID(), initialCheckpointId)) + .isGreaterThan(initialCheckpointId); + retry( + DEFAULT_TIMEOUT, + () -> assertThat(lookupLong(tablePath, 1L, 1)).as("Recovered MAX").isEqualTo(20L)); + assertProducerOffsetsAbsent(producerId); - // Cancel the job after verification job.cancel().get(); waitForJobTermination(job, DEFAULT_TIMEOUT); - - // Final verification - Long[] finalSums = lookupSumsForKeys(tablePath, 1L, 2L, 3L); - LOG.info( - "Final sums: key1={}, key2={}, key3={}, expected={}", - finalSums[0], - finalSums[1], - finalSums[2], - expectedFinalSumPerKey); - - for (int i = 0; i < 3; i++) { - assertThat(finalSums[i]) - .as("Key " + (i + 1) + " final sum") - .isEqualTo(expectedFinalSumPerKey); - } + assertThat(lookupLong(tablePath, 1L, 1)).isEqualTo(20L); } - /** - * Tests producer offset recovery when failure occurs before any checkpoint completes. - * - *

This test verifies that undo recovery works even without checkpoint state, using the - * producer offset stored on the server side. The test uses FailingCountingSource configured to - * fail quickly (after just 2 records) before a checkpoint can complete. - * - *

Pattern: - * - *

    - *
  1. Write 2 records (failure triggers before checkpoint) - *
  2. Failure is triggered - *
  3. Auto-recover from fresh state (no checkpoint to restore from) - *
  4. Producer offset recovery undoes the 2 pre-failure records - *
  5. Source emits total of 7 records (2+5), but only 7 are persisted after undo - *
- * - *

Expected result: Final sum = 7 * VALUE_PER_RECORD = 70 - * - *

Note: Even without checkpoint state, Fluss can perform undo recovery using the producer - * offset stored on the server. The pre-failure writes ARE undone because the producer ID allows - * the server to identify uncommitted writes from the previous run. - */ + /** Tests producer-offset recovery before the first checkpoint. */ @Test void testProducerOffsetRecoveryWithFailover() throws Exception { String tableName = "undo_producer_offset_" + System.currentTimeMillis(); TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); + String producerId = "undo-before-checkpoint-" + System.nanoTime(); - initTableEnvironment(null, false).executeSql(createAggTableDDL(tableName)); - - // Configure to fail quickly before checkpoint completes - int recordsBeforeFailure = 2; - int recordsAfterFailure = 5; - - // Start job with FailingCountingSource (default producerId = Flink Job ID) - JobClient job = - startFailoverJob( - tablePath, null, recordsBeforeFailure, recordsAfterFailure, 1, false); + initTableEnvironment(null, false) + .executeSql(createAggTableDDL(tableName, DEFAULT_BUCKET_NUM, "allow")); + FailingCountingSource source = + FailingCountingSource.coordinatedSingleKey( + failureMarkerDir, 1L, 10L, 0, 10L, 2, 10L, 5); + JobClient job = startCoordinatedFailoverDataStreamJob(tablePath, producerId, source); - // Wait for final sum after recovery - // Expected: 7 * VALUE_PER_RECORD = 70 - // Pre-failure writes ARE undone via producer offset recovery - long expectedFinalSum = (recordsBeforeFailure + recordsAfterFailure) * VALUE_PER_RECORD; + long initialLogEndOffset = getLatestLogEndOffset(tablePath, 0); + source.releaseDirtyEmission(); + waitForLogEndOffsetAtLeast(tablePath, 0, initialLogEndOffset + 2); retry( DEFAULT_TIMEOUT, - () -> { - Long sum = lookupSum(tablePath, 1L); - assertThat(sum) - .as("Final sum after producer offset recovery") - .isEqualTo(expectedFinalSum); - }); + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Dirty writes before the first checkpoint") + .isEqualTo(20L)); + assertThat(findLatestCompletedCheckpoint(job.getJobID().toHexString())) + .as("The injected failure must precede every completed checkpoint") + .isEqualTo(-1L); + assertRecoveryActionObservable(tablePath, producerId, RecoveryAction.UNDO); + + triggerFailureAndWaitForRecovery(source); + long firstCheckpointId = triggerCompletedCheckpointAfter(job.getJobID(), -1L); + assertThat(firstCheckpointId).isGreaterThanOrEqualTo(0L); + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(lookupSum(tablePath, 1L)) + .as("Stable SUM after producer-offset recovery") + .isEqualTo(70L)); - // Cancel the job after verification job.cancel().get(); waitForJobTermination(job, DEFAULT_TIMEOUT); - - // Final verification - Long finalSum = lookupSum(tablePath, 1L); - LOG.info("Final sum: {}, expected: {}", finalSum, expectedFinalSum); - assertThat(finalSum) - .as( - "Final sum should equal (recordsBeforeFailure + recordsAfterFailure) * VALUE_PER_RECORD") - .isEqualTo(expectedFinalSum); + assertThat(lookupSum(tablePath, 1L)).isEqualTo(70L); + admin.deleteProducerOffsets(producerId).get(); } /** Tests Rescale Up - undo recovery when parallelism increases (1 -> 2). */ @@ -499,7 +451,7 @@ void testSqlInsertWithUndoRecovery() throws Exception { // Phase 1: Write dirty data via DataStream API with the same producerId, cancel without // checkpoint - JobClient dirtyJob = startBoundedJob(tablePath, producerId, null, 5, 1, false); + JobClient dirtyJob = startBoundedJob(tablePath, producerId, null, 5, 1, false, false); // Wait for dirty data to be written long dirtySum = 5 * VALUE_PER_RECORD; // 50 @@ -544,32 +496,25 @@ void testSqlInsertWithUndoRecovery() throws Exception { }); // Verify producer offsets cleaned up - verifyProducerOffsetsCleanedUp(producerId); + assertProducerOffsetsAbsent(producerId); } /** * Tests that producer offsets are cleaned up via endInput() for bounded jobs without * checkpointing. * - *

This verifies the cleanup path added in Comment #12: when a bounded job finishes (source - * reaches end of input) without any checkpoint completing, {@code endInput()} calls {@code - * deleteProducerOffsetsIfNeeded()} to ensure producer offsets don't remain in ZK. + *

When a bounded source reaches end of input without a completed checkpoint, {@code + * endInput()} still removes its producer offsets. */ @Test void testBoundedJobCleansUpProducerOffsetsWithoutCheckpoint() throws Exception { String tableName = "undo_bounded_cleanup_" + System.currentTimeMillis(); - TablePath tablePath = TablePath.of(DEFAULT_DB, tableName); String producerId = "test-producer-bounded-" + System.currentTimeMillis(); - // Create table via SQL DDL StreamTableEnvironment tEnv = initTableEnvironment(null, false); - tEnv.executeSql(createAggTableDDL(tableName)); - - // Use SQL INSERT which is inherently bounded — no checkpointing needed. - // endInput() is called when the bounded source finishes. tEnv.executeSql( String.format( - "CREATE TABLE `%s`.`%s_with_pid` (" + "CREATE TABLE `%s`.`%s` (" + " id BIGINT NOT NULL PRIMARY KEY NOT ENFORCED," + " sum_val BIGINT" + ") WITH (" @@ -583,12 +528,12 @@ void testBoundedJobCleansUpProducerOffsetsWithoutCheckpoint() throws Exception { // Execute bounded SQL INSERT (no checkpointing enabled in initTableEnvironment) tEnv.executeSql( String.format( - "INSERT INTO `%s`.`%s_with_pid` VALUES (1, 10), (1, 20), (1, 30)", + "INSERT INTO `%s`.`%s` VALUES (1, 10), (1, 20), (1, 30)", DEFAULT_DB, tableName)) .await(); // Verify data written - TablePath actualPath = TablePath.of(DEFAULT_DB, tableName + "_with_pid"); + TablePath actualPath = TablePath.of(DEFAULT_DB, tableName); retry( DEFAULT_TIMEOUT, () -> { @@ -597,11 +542,81 @@ void testBoundedJobCleansUpProducerOffsetsWithoutCheckpoint() throws Exception { }); // Verify producer offsets cleaned up via endInput() path - verifyProducerOffsetsCleanedUp(producerId); + assertProducerOffsetsAbsent(producerId); } // ==================== Reusable Test Patterns ==================== + private void triggerFailureAndWaitForRecovery(FailingCountingSource source) throws Exception { + source.triggerFailure(); + waitUntil( + source::hasFailed, + DEFAULT_TIMEOUT, + "Timeout waiting for the source failure marker"); + waitUntil( + source::hasRestarted, + DEFAULT_TIMEOUT, + "Timeout waiting for the source restart marker"); + waitUntil( + source::hasRecoveryEmissionCompleted, + DEFAULT_TIMEOUT, + "Timeout waiting for complete recovery emission"); + } + + private void waitForLogEndOffsetAtLeast( + TablePath tablePath, int bucketId, long minimumLogEndOffset) { + retry( + DEFAULT_TIMEOUT, + () -> + assertThat(getLatestLogEndOffset(tablePath, bucketId)) + .as("Records acknowledged in the Fluss log") + .isGreaterThanOrEqualTo(minimumLogEndOffset)); + } + + private void assertRecoveryActionObservable( + TablePath tablePath, String producerId, RecoveryAction expectedRecoveryAction) + throws Exception { + if (expectedRecoveryAction == RecoveryAction.NO_OP) { + assertThat(admin.getProducerOffsets(producerId).get()) + .as("NO_OP must not register producer offsets") + .isNull(); + return; + } + + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + TableBucket expectedBucket = new TableBucket(tableId, 0); + retry( + DEFAULT_TIMEOUT, + () -> { + ProducerOffsetsResult result = admin.getProducerOffsets(producerId).get(); + assertThat(result).as("UNDO must register producer offsets").isNotNull(); + assertThat(result.getProducerId()).isEqualTo(producerId); + assertThat(result.getTableOffsets()).containsOnlyKeys(tableId); + assertThat(result.getTableOffsets().get(tableId)) + .containsOnlyKeys(expectedBucket); + assertThat(result.getTableOffsets().get(tableId).get(expectedBucket)) + .isGreaterThanOrEqualTo(0L); + }); + } + + private long getLatestLogEndOffset(TablePath tablePath, int bucketId) throws Exception { + try { + Map latestOffsets = + admin.listOffsets( + tablePath, + Collections.singleton(bucketId), + new OffsetSpec.LatestSpec()) + .all() + .get(DEFAULT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + assertThat(latestOffsets).containsOnlyKeys(bucketId); + assertThat(latestOffsets.get(bucketId)).isNotNull(); + return latestOffsets.get(bucketId); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; + } + } + /** * Runs a three-phase undo recovery test with multiple buckets and multiple keys. * @@ -637,7 +652,13 @@ private void runThreePhaseMultiBucketUndoTest( // Phase 1: Write records for multiple keys and take savepoint JobClient phase1Job = startBoundedJob( - tablePath, producerId, null, recordsPerPhase[0], phase12Parallelism, true); + tablePath, + producerId, + null, + recordsPerPhase[0], + phase12Parallelism, + true, + true); // Wait for data to be written before checkpoint (verify all 3 keys) long expectedSumPerKey = recordsPerPhase[0] * VALUE_PER_RECORD; @@ -679,6 +700,7 @@ private void runThreePhaseMultiBucketUndoTest( savepointPath, recordsPerPhase[1], phase12Parallelism, + true, true); // Wait for Phase 2 data to be written (verify all 3 keys) @@ -713,6 +735,7 @@ private void runThreePhaseMultiBucketUndoTest( savepointPath, recordsPerPhase[2], phase3Parallelism, + true, true); // Wait for Phase 3 data to be written (after undo, verify all 3 keys) @@ -729,9 +752,9 @@ private void runThreePhaseMultiBucketUndoTest( } }); - // Verify producer offsets have been cleaned up from ZK after checkpoint (Comment #8) + // Verify producer offsets have been cleaned up after checkpoint. waitForCheckpoint(phase3Job.getJobID()); - verifyProducerOffsetsCleanedUp(producerId); + assertProducerOffsetsAbsent(producerId); phase3Job.cancel().get(); waitForJobTermination(phase3Job, DEFAULT_TIMEOUT); @@ -763,11 +786,16 @@ private void runThreePhaseMultiBucketUndoTest( // ==================== Helper Methods ==================== - private String createAggTableDDL(String tableName) { - return createAggTableDDL(tableName, DEFAULT_BUCKET_NUM); + private String createAggTableDDL(String tableName, int bucketNum) { + return createAggTableDDL(tableName, bucketNum, null); } - private String createAggTableDDL(String tableName, int bucketNum) { + private String createAggTableDDL( + String tableName, int bucketNum, @Nullable String deleteBehavior) { + String deleteBehaviorOption = + deleteBehavior == null + ? "" + : String.format(", 'table.delete.behavior' = '%s'", deleteBehavior); return String.format( "CREATE TABLE `%s`.`%s` (" + " id BIGINT NOT NULL PRIMARY KEY NOT ENFORCED," @@ -776,8 +804,43 @@ private String createAggTableDDL(String tableName, int bucketNum) { + " 'bucket.num' = '%d'," + " 'table.merge-engine' = 'aggregation'," + " 'fields.sum_val.agg' = 'sum'" + + "%s" + + ")", + DEFAULT_DB, tableName, bucketNum, deleteBehaviorOption); + } + + private String createMaxAggTableDDL( + String tableName, String producerId, String deleteBehavior) { + return String.format( + "CREATE TABLE `%s`.`%s` (" + + " id BIGINT NOT NULL PRIMARY KEY NOT ENFORCED," + + " max_value BIGINT" + + ") WITH (" + + " 'bucket.num' = '%d'," + + " 'table.merge-engine' = 'aggregation'," + + " 'fields.max_value.agg' = 'max'," + + " 'sink.producer-id' = '%s'," + + " 'table.delete.behavior' = '%s'" + ")", - DEFAULT_DB, tableName, bucketNum); + DEFAULT_DB, tableName, DEFAULT_BUCKET_NUM, producerId, deleteBehavior); + } + + private String createMixedAggTableDDL( + String tableName, String producerId, String deleteBehavior) { + return String.format( + "CREATE TABLE `%s`.`%s` (" + + " id BIGINT NOT NULL PRIMARY KEY NOT ENFORCED," + + " max_value BIGINT," + + " sum_value BIGINT" + + ") WITH (" + + " 'bucket.num' = '%d'," + + " 'table.merge-engine' = 'aggregation'," + + " 'fields.max_value.agg' = 'max'," + + " 'fields.sum_value.agg' = 'sum'," + + " 'sink.producer-id' = '%s'," + + " 'table.delete.behavior' = '%s'" + + ")", + DEFAULT_DB, tableName, DEFAULT_BUCKET_NUM, producerId, deleteBehavior); } /** @@ -789,6 +852,7 @@ private String createAggTableDDL(String tableName, int bucketNum) { * @param maxRecords max records to emit (per key if multiKey) * @param parallelism job parallelism * @param multiKey if true, emit to keys 1,2,3 in round-robin + * @param enableCheckpointing whether to enable periodic checkpoints */ private JobClient startBoundedJob( TablePath tablePath, @@ -796,7 +860,8 @@ private JobClient startBoundedJob( @Nullable String savepointPath, int maxRecords, int parallelism, - boolean multiKey) + boolean multiKey, + boolean enableCheckpointing) throws Exception { Configuration conf = new Configuration(); @@ -806,7 +871,9 @@ private JobClient startBoundedJob( StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); env.setParallelism(parallelism); - env.enableCheckpointing(1000); + if (enableCheckpointing) { + env.enableCheckpointing(1000); + } CountingSource source = multiKey @@ -832,109 +899,44 @@ private JobClient startBoundedJob( return env.executeAsync(jobName); } - /** - * Starts a job with FailingCountingSource for checkpoint-based failover testing. - * - *

This method configures a job that will: - * - *

    - *
  1. Emit {@code recordsBeforeFailure} records - *
  2. Trigger a failure (RuntimeException) - *
  3. Auto-recover from checkpoint and emit {@code recordsAfterFailure} records - *
- * - * @param tablePath target table - * @param producerId producer ID for undo recovery, or null to use default (Flink Job ID) - * @param recordsBeforeFailure records to emit before triggering failure - * @param recordsAfterFailure records to emit after recovery - * @param parallelism job parallelism - * @param multiKey if true, emit to keys 1,2,3 in round-robin - * @return JobClient for monitoring the job - */ - private JobClient startFailoverJob( - TablePath tablePath, - @Nullable String producerId, - int recordsBeforeFailure, - int recordsAfterFailure, - int parallelism, - boolean multiKey) - throws Exception { - - Configuration conf = new Configuration(); - // No savepoint path - this is for checkpoint-based failover - - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); - env.setParallelism(parallelism); - env.enableCheckpointing(1000); - - FailingCountingSource source = - multiKey - ? FailingCountingSource.multiKey( - failureMarkerDir, - VALUE_PER_RECORD, - recordsBeforeFailure, - recordsAfterFailure) - : FailingCountingSource.singleKey( - failureMarkerDir, - 1L, - VALUE_PER_RECORD, - recordsBeforeFailure, - recordsAfterFailure); + private JobClient startCoordinatedFailoverDataStreamJob( + TablePath tablePath, String producerId, FailingCountingSource source) throws Exception { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + env.enableCheckpointing(TimeUnit.DAYS.toMillis(1)); DataStreamSource stream = - env.fromSource(source, WatermarkStrategy.noWatermarks(), "failing-counting-source"); - - FlussSink sink; - FlussSinkBuilder sinkBuilder = + env.fromSource( + source, + WatermarkStrategy.noWatermarks(), + "failing-counting-source", + source.getProducedType()); + FlussSink sink = FlussSink.builder() .setBootstrapServers(bootstrapServers) .setDatabase(tablePath.getDatabaseName()) .setTable(tablePath.getTableName()) - .setSerializationSchema(new RowDataSerializationSchema(false, true)); - if (producerId != null) { - sinkBuilder.setProducerId(producerId); - } - sink = sinkBuilder.build(); - + .setProducerId(producerId) + .setSerializationSchema(new RowDataSerializationSchema(false, true)) + .build(); stream.sinkTo(sink).name("Fluss Sink"); - - String jobName = - multiKey ? "Multi-Key Failover Test" : "Failover Test (p=" + parallelism + ")"; - return env.executeAsync(jobName); + return env.executeAsync("Coordinated aggregation failover"); } - /** - * Starts a job with FailingCountingSource for checkpoint-based failover testing. - * - *

This method configures a job that will: - * - *

    - *
  1. Emit {@code recordsBeforeFailure} records - *
  2. Trigger a failure (RuntimeException) - *
  3. Auto-recover from checkpoint and emit {@code recordsAfterFailure} records - *
- * - * @param tablePath target table - * @param recordsBeforeFailure records to emit before triggering failure - * @param recordsAfterFailure records to emit after recovery - * @param parallelism job parallelism - * @param multiKey if true, emit to keys 1,2,3 in round-robin - * @return JobClient for monitoring the job - */ private TableResult startFailoverSqlJob( TablePath tablePath, String ddl, - int recordsBeforeFailure, - int recordsAfterFailure, int parallelism, - boolean multiKey) + @Nullable List targetColumns, + FailingCountingSource source, + long checkpointInterval) throws Exception { Configuration conf = new Configuration(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf); env.setParallelism(parallelism); - env.enableCheckpointing(1000); + env.enableCheckpointing(checkpointInterval); StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, EnvironmentSettings.inStreamingMode()); @@ -945,20 +947,6 @@ private TableResult startFailoverSqlJob( CATALOG_NAME, BOOTSTRAP_SERVERS.key(), bootstrapServers)); tEnv.executeSql("USE CATALOG " + CATALOG_NAME); - FailingCountingSource source = - multiKey - ? FailingCountingSource.multiKey( - failureMarkerDir, - VALUE_PER_RECORD, - recordsBeforeFailure, - recordsAfterFailure) - : FailingCountingSource.singleKey( - failureMarkerDir, - 1L, - VALUE_PER_RECORD, - recordsBeforeFailure, - recordsAfterFailure); - DataStreamSource stream = env.fromSource( source, @@ -976,23 +964,29 @@ private TableResult startFailoverSqlJob( tEnv.executeSql(ddl).await(); + String targetClause = + targetColumns == null + ? "" + : targetColumns.stream() + .map(column -> "`" + column + "`") + .collect(Collectors.joining(", ", " (", ")")); + String sourceQuery = + targetColumns == null + ? "SELECT * FROM source_table" + : "SELECT `key`, `value` FROM source_table"; return tEnv.executeSql( "INSERT INTO `" + tablePath.getDatabaseName() + "`.`" + tablePath.getTableName() - + "` SELECT * FROM source_table"); + + "`" + + targetClause + + " " + + sourceQuery); } - /** - * Verifies that producer offsets have been cleaned up from ZK. - * - *

After a successful checkpoint, the UndoRecoveryOperator deletes producer offsets from ZK. - * This method retries the check to account for async cleanup timing. - * - * @param producerId the producer ID to check - */ - private void verifyProducerOffsetsCleanedUp(String producerId) { + /** Verifies that no producer-offset registration remains for the given producer. */ + private void assertProducerOffsetsAbsent(String producerId) { retry( DEFAULT_TIMEOUT, () -> { @@ -1009,10 +1003,15 @@ private void verifyProducerOffsetsCleanedUp(String producerId) { @Nullable private Long lookupSum(TablePath tablePath, Long key) throws Exception { + return lookupLong(tablePath, key, 1); + } + + @Nullable + private Long lookupLong(TablePath tablePath, Long key, int fieldIndex) throws Exception { try (Table table = conn.getTable(tablePath)) { Lookuper lookuper = table.newLookup().createLookuper(); InternalRow result = lookuper.lookup(row(key)).get().getSingletonRow(); - return result == null ? null : result.getLong(1); + return result == null ? null : result.getLong(fieldIndex); } } @@ -1068,31 +1067,79 @@ protected static Configuration getFileBasedCheckpointsConfig() { protected void waitForJobTermination(JobClient jobClient, Duration timeout) { waitUntil( - () -> { - try { - return jobClient.getJobStatus().get().isTerminalState(); - } catch (Exception e) { - return true; - } - }, + () -> jobClient.getJobStatus().get().isTerminalState(), timeout, "Timeout waiting for job termination"); } protected void waitForCheckpoint(JobID jobId) { + waitForCompletedCheckpointAfter(jobId, -1L); + } + + private long triggerCompletedCheckpointAfter(JobID jobId, long minimumCheckpointId) + throws Exception { + long deadlineNanos = System.nanoTime() + DEFAULT_TIMEOUT.toNanos(); + while (true) { + waitUntilAllTasksAreRunning(miniCluster.getRestClusterClient(), jobId); + long remainingMillis = + Math.max(1L, TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime())); + try { + miniCluster + .getMiniCluster() + .triggerCheckpoint(jobId) + .get(remainingMillis, TimeUnit.MILLISECONDS); + return waitForCompletedCheckpointAfter(jobId, minimumCheckpointId); + } catch (ExecutionException e) { + if (!isTaskReadinessCheckpointFailure(e) || System.nanoTime() >= deadlineNanos) { + throw e; + } + LOG.info("Checkpoint raced with a task restart; retrying when all tasks run"); + Thread.sleep(50L); + } + } + } + + private boolean isTaskReadinessCheckpointFailure(ExecutionException exception) { + Throwable cause = exception.getCause(); + return cause instanceof CheckpointException + && ((CheckpointException) cause).getCheckpointFailureReason() + == CheckpointFailureReason.NOT_ALL_REQUIRED_TASKS_RUNNING; + } + + private long waitForCompletedCheckpointAfter(JobID jobId, long minimumCheckpointId) { String jobIdStr = jobId.toHexString(); waitUntil( - () -> { - File jobCheckpointDir = new File(checkpointDir, jobIdStr); - if (!jobCheckpointDir.exists()) { - return false; - } - File[] checkpoints = - jobCheckpointDir.listFiles( - f -> f.isDirectory() && f.getName().startsWith("chk-")); - return checkpoints != null && checkpoints.length > 0; - }, + () -> findLatestCompletedCheckpoint(jobIdStr) > minimumCheckpointId, DEFAULT_TIMEOUT, - "Timeout waiting for checkpoint for job " + jobIdStr); + "Timeout waiting for a completed checkpoint after " + + minimumCheckpointId + + " for job " + + jobIdStr); + return findLatestCompletedCheckpoint(jobIdStr); + } + + private long findLatestCompletedCheckpoint(String jobIdStr) { + File jobCheckpointDir = new File(checkpointDir, jobIdStr); + File[] checkpoints = + jobCheckpointDir.listFiles( + file -> + file.isDirectory() + && file.getName().startsWith("chk-") + && new File(file, "_metadata").isFile()); + long latestCheckpointId = -1L; + if (checkpoints != null) { + for (File checkpoint : checkpoints) { + try { + latestCheckpointId = + Math.max( + latestCheckpointId, + Long.parseLong( + checkpoint.getName().substring("chk-".length()))); + } catch (NumberFormatException ignored) { + // Ignore unrelated directories with a chk- prefix. + } + } + } + return latestCheckpointId; } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/FailingCountingSource.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/FailingCountingSource.java index 8d29bc42ee..c874ea97cd 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/FailingCountingSource.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/testutils/FailingCountingSource.java @@ -37,34 +37,16 @@ import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; -import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Queue; import java.util.concurrent.CompletableFuture; -/** - * A counting source that can inject failures at specified points for testing checkpoint-based - * failover recovery. - * - *

This source emits records like CountingSource but throws a RuntimeException after emitting a - * configured number of records. The failure is "one-shot" - after recovery from checkpoint, the - * source continues without failing again. - * - *

The source operates in three phases: - * - *

    - *
  1. Phase 1: Emit {@code recordsBeforeFailure} records - *
  2. Phase 2: Throw RuntimeException to trigger job failover - *
  3. Phase 3: After recovery, emit {@code recordsAfterFailure} records - *
- * - *

The {@code hasFailed} flag is tracked using a file-based marker. This allows failure state to - * persist across Enumerator and Reader restarts, even when different ClassLoaders are used. - */ +/** A deterministic single-key source for checkpoint failover tests. */ public class FailingCountingSource implements Source< RowData, @@ -72,93 +54,100 @@ public class FailingCountingSource FailingCountingSource.FailingEnumeratorState>, ResultTypeQueryable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; + private static final long MARKER_POLL_INTERVAL_MILLIS = 10L; + private static final String DIRTY_RELEASE_MARKER_SUFFIX = ".release-dirty"; + private static final String FAILURE_TRIGGER_MARKER_SUFFIX = ".trigger-failure"; + private static final String FAILURE_MARKER_SUFFIX = ".failed"; + private static final String RESTART_MARKER_SUFFIX = ".restarted"; + private static final String RECOVERY_COMPLETE_MARKER_SUFFIX = ".recovery-complete"; + private static final CompletableFuture AVAILABLE = + CompletableFuture.completedFuture(null); - // Unique identifier for this source instance private final String sourceId; - // Directory for failure marker files (passed from test, survives ClassLoader changes) private final String markerDirPath; private final long key; - private final long value; - private final int recordsBeforeFailure; - private final int recordsAfterFailure; - private final boolean multiKey; + private final long committedValue; + private final int committedRecords; + private final long dirtyValue; + private final int dirtyRecords; + private final long recoveryValue; + private final int recoveryRecords; private FailingCountingSource( String sourceId, String markerDirPath, long key, - long value, - int recordsBeforeFailure, - int recordsAfterFailure, - boolean multiKey) { + long committedValue, + int committedRecords, + long dirtyValue, + int dirtyRecords, + long recoveryValue, + int recoveryRecords) { + if (committedRecords < 0 || dirtyRecords < 0 || recoveryRecords <= 0) { + throw new IllegalArgumentException( + "Committed and dirty record counts must not be negative, and recovery records must be positive"); + } this.sourceId = sourceId; this.markerDirPath = markerDirPath; this.key = key; - this.value = value; - this.recordsBeforeFailure = recordsBeforeFailure; - this.recordsAfterFailure = recordsAfterFailure; - this.multiKey = multiKey; + this.committedValue = committedValue; + this.committedRecords = committedRecords; + this.dirtyValue = dirtyValue; + this.dirtyRecords = dirtyRecords; + this.recoveryValue = recoveryValue; + this.recoveryRecords = recoveryRecords; } - /** - * Creates a single-key source that emits records for the specified key with failure injection. - * - * @param markerDir directory for failure marker files (use @TempDir for test isolation) - * @param key the key to emit records for - * @param value the value for each record - * @param recordsBeforeFailure number of records to emit before triggering failure - * @param recordsAfterFailure number of records to emit after recovery - * @return a new FailingCountingSource instance - */ - public static FailingCountingSource singleKey( - java.io.File markerDir, + /** Creates a source with independently controlled committed, dirty, and recovery phases. */ + public static FailingCountingSource coordinatedSingleKey( + File markerDir, long key, - long value, - int recordsBeforeFailure, - int recordsAfterFailure) { - String sourceId = "single-" + key + "-" + System.nanoTime(); + long committedValue, + int committedRecords, + long dirtyValue, + int dirtyRecords, + long recoveryValue, + int recoveryRecords) { return new FailingCountingSource( - sourceId, + "coordinated-single-" + key + "-" + System.nanoTime(), markerDir.getAbsolutePath(), key, - value, - recordsBeforeFailure, - recordsAfterFailure, - false); + committedValue, + committedRecords, + dirtyValue, + dirtyRecords, + recoveryValue, + recoveryRecords); } - /** - * Creates a multi-key source that emits records for keys 1, 2, 3 in round-robin with failure - * injection. - * - * @param markerDir directory for failure marker files (use @TempDir for test isolation) - * @param value the value for each record - * @param recordsBeforeFailurePerKey number of records per key to emit before triggering failure - * @param recordsAfterFailurePerKey number of records per key to emit after recovery - * @return a new FailingCountingSource instance - */ - public static FailingCountingSource multiKey( - java.io.File markerDir, - long value, - int recordsBeforeFailurePerKey, - int recordsAfterFailurePerKey) { - String sourceId = "multi-" + System.nanoTime(); - return new FailingCountingSource( - sourceId, - markerDir.getAbsolutePath(), - 0, - value, - recordsBeforeFailurePerKey, - recordsAfterFailurePerKey, - true); + /** Releases the source's dirty suffix. */ + public void releaseDirtyEmission() throws IOException { + createMarker(DIRTY_RELEASE_MARKER_SUFFIX); + } + + /** Triggers the source's one-shot failure. */ + public void triggerFailure() throws IOException { + createMarker(FAILURE_TRIGGER_MARKER_SUFFIX); + } + + /** Returns whether the source has injected its failure. */ + public boolean hasFailed() { + return markerExists(FAILURE_MARKER_SUFFIX); + } + + /** Returns whether the failed reader has restarted. */ + public boolean hasRestarted() { + return markerExists(RESTART_MARKER_SUFFIX); + } + + /** Returns whether all post-restart records have been emitted. */ + public boolean hasRecoveryEmissionCompleted() { + return markerExists(RECOVERY_COMPLETE_MARKER_SUFFIX); } @Override public Boundedness getBoundedness() { - // CONTINUOUS_UNBOUNDED allows the source to idle after emitting records - // This is needed for checkpoint tests where we want checkpoints to complete - // after all records are emitted but before the job terminates return Boundedness.CONTINUOUS_UNBOUNDED; } @@ -166,19 +155,14 @@ public Boundedness getBoundedness() { public SplitEnumerator createEnumerator( SplitEnumeratorContext context) { return new FailingCountingSplitEnumerator( - context, recordsBeforeFailure, recordsAfterFailure, multiKey, false); + context, new FailingEnumeratorState(new FailingCountingSplit(0, 0))); } @Override public SplitEnumerator restoreEnumerator( SplitEnumeratorContext context, FailingEnumeratorState checkpoint) { - return new FailingCountingSplitEnumerator( - context, - checkpoint.getRecordsBeforeFailure(), - checkpoint.getRecordsAfterFailure(), - checkpoint.isMultiKey(), - checkpoint.isSplitAssigned()); + return new FailingCountingSplitEnumerator(context, checkpoint); } @Override @@ -198,10 +182,12 @@ public SourceReader createReader( sourceId, markerDirPath, key, - value, - multiKey, - recordsBeforeFailure, - recordsAfterFailure); + committedValue, + committedRecords, + dirtyValue, + dirtyRecords, + recoveryValue, + recoveryRecords); } @Override @@ -215,345 +201,242 @@ public TypeInformation getProducedType() { return InternalTypeInfo.of(rowType); } - // ==================== FailingCountingSplit ==================== - - /** - * Split state for FailingCountingSource that tracks failure status. - * - *

This split contains: - * - *

    - *
  • {@code splitId} - unique identifier for the split - *
  • {@code emittedRecords} - number of records emitted so far - *
  • {@code hasFailed} - whether the failure has already been triggered (one-shot flag) - *
- */ + private boolean markerExists(String suffix) { + return new File(markerDirPath, sourceId + suffix).isFile(); + } + + private void createMarker(String suffix) throws IOException { + createMarker(markerDirPath, sourceId + suffix); + } + + private static void createMarker(String markerDirPath, String markerName) throws IOException { + File markerDir = new File(markerDirPath); + if (!markerDir.isDirectory() && !markerDir.mkdirs() && !markerDir.isDirectory()) { + throw new IOException("Failed to create marker directory " + markerDir); + } + File marker = new File(markerDir, markerName); + if (!marker.createNewFile() && !marker.isFile()) { + throw new IOException("Failed to create marker " + marker); + } + } + + /** Checkpointed progress for the source's single split. */ public static class FailingCountingSplit implements SourceSplit, Serializable { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; private final int splitId; private final int emittedRecords; - private final boolean hasFailed; - private final int keyIndex; - public FailingCountingSplit(int splitId, int emittedRecords, boolean hasFailed) { - this(splitId, emittedRecords, hasFailed, 0); - } - - public FailingCountingSplit( - int splitId, int emittedRecords, boolean hasFailed, int keyIndex) { + private FailingCountingSplit(int splitId, int emittedRecords) { this.splitId = splitId; this.emittedRecords = emittedRecords; - this.hasFailed = hasFailed; - this.keyIndex = keyIndex; } @Override public String splitId() { return String.valueOf(splitId); } - - public int getSplitIdInt() { - return splitId; - } - - public int getEmittedRecords() { - return emittedRecords; - } - - public boolean hasFailed() { - return hasFailed; - } - - public int getKeyIndex() { - return keyIndex; - } - - @Override - public String toString() { - return "FailingCountingSplit{" - + "splitId=" - + splitId - + ", emittedRecords=" - + emittedRecords - + ", hasFailed=" - + hasFailed - + ", keyIndex=" - + keyIndex - + '}'; - } } - // ==================== FailingEnumeratorState ==================== - - /** - * Enumerator state for FailingCountingSource. - * - *

This state contains configuration needed to restore the enumerator: - * - *

    - *
  • {@code recordsBeforeFailure} - records to emit before failure - *
  • {@code recordsAfterFailure} - records to emit after recovery - *
  • {@code multiKey} - whether multi-key mode is enabled - *
  • {@code splitAssigned} - whether a split has been assigned - *
- */ + /** Checkpointed assignment for the source's single split. */ public static class FailingEnumeratorState implements Serializable { - private static final long serialVersionUID = 1L; - - private final int recordsBeforeFailure; - private final int recordsAfterFailure; - private final boolean multiKey; - private final boolean splitAssigned; - - public FailingEnumeratorState( - int recordsBeforeFailure, - int recordsAfterFailure, - boolean multiKey, - boolean splitAssigned) { - this.recordsBeforeFailure = recordsBeforeFailure; - this.recordsAfterFailure = recordsAfterFailure; - this.multiKey = multiKey; - this.splitAssigned = splitAssigned; - } - - public int getRecordsBeforeFailure() { - return recordsBeforeFailure; - } - - public int getRecordsAfterFailure() { - return recordsAfterFailure; - } + private static final long serialVersionUID = 2L; - public boolean isMultiKey() { - return multiKey; - } + @Nullable private final FailingCountingSplit pendingSplit; - public boolean isSplitAssigned() { - return splitAssigned; + private FailingEnumeratorState(@Nullable FailingCountingSplit pendingSplit) { + this.pendingSplit = pendingSplit; } } - // ==================== FailingCountingSplitEnumerator ==================== - - /** - * Split enumerator for FailingCountingSource. - * - *

This enumerator assigns a single split to the first reader (subtask 0). - */ private static class FailingCountingSplitEnumerator implements SplitEnumerator { private final SplitEnumeratorContext context; - private final int recordsBeforeFailure; - private final int recordsAfterFailure; - private final boolean multiKey; - private boolean splitAssigned; + @Nullable private FailingCountingSplit pendingSplit; - FailingCountingSplitEnumerator( + private FailingCountingSplitEnumerator( SplitEnumeratorContext context, - int recordsBeforeFailure, - int recordsAfterFailure, - boolean multiKey, - boolean splitAssigned) { + FailingEnumeratorState checkpoint) { this.context = context; - this.recordsBeforeFailure = recordsBeforeFailure; - this.recordsAfterFailure = recordsAfterFailure; - this.multiKey = multiKey; - this.splitAssigned = splitAssigned; + this.pendingSplit = checkpoint.pendingSplit; } @Override public void start() {} @Override - public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { - // Not used - we assign splits proactively - } + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) {} @Override public void addSplitsBack(List splits, int subtaskId) { - // Re-assign splits if a reader fails - if (!splits.isEmpty()) { - splitAssigned = false; + if (splits.size() != 1) { + throw new IllegalArgumentException( + "Expected one returned split, but got " + splits.size()); } + pendingSplit = splits.get(0); + assignPendingSplit(subtaskId); } @Override public void addReader(int subtaskId) { - if (!splitAssigned && subtaskId == 0) { - // Assign a fresh split - failure state is tracked in static map - context.assignSplit(new FailingCountingSplit(0, 0, false), subtaskId); - splitAssigned = true; + assignPendingSplit(subtaskId); + } + + private void assignPendingSplit(int subtaskId) { + if (subtaskId == 0 + && pendingSplit != null + && context.registeredReaders().containsKey(subtaskId)) { + context.assignSplit(pendingSplit, subtaskId); + pendingSplit = null; } } @Override public FailingEnumeratorState snapshotState(long checkpointId) { - return new FailingEnumeratorState( - recordsBeforeFailure, recordsAfterFailure, multiKey, splitAssigned); + return new FailingEnumeratorState(pendingSplit); } @Override public void close() {} } - // ==================== FailingCountingReader ==================== - - /** - * Reader that emits records and triggers failure at the configured point. - * - *

The reader tracks: - * - *

    - *
  • {@code totalEmitted} - total records emitted across all phases - *
  • {@code hasFailed} - whether the failure has been triggered (one-shot) - *
- * - *

Uses file-based markers to track failure across ClassLoader restarts. - */ private static class FailingCountingReader implements SourceReader { private final String sourceId; private final String markerDirPath; - private final long singleKey; - private final long value; - private final boolean multiKey; - private final int recordsBeforeFailure; - private final int recordsAfterFailure; - private final Queue splits = new ArrayDeque<>(); - private CompletableFuture availableFuture = new CompletableFuture<>(); - - // State tracked across checkpoints - private int totalEmitted = 0; - private boolean hasFailed = false; - private int keyIndex = 0; - - FailingCountingReader( + private final long key; + private final long committedValue; + private final int committedRecords; + private final long dirtyValue; + private final int dirtyRecords; + private final long recoveryValue; + private final int recoveryRecords; + private final CompletableFuture splitAvailability = new CompletableFuture<>(); + + @Nullable private FailingCountingSplit activeSplit; + private int emittedRecords; + + private FailingCountingReader( String sourceId, String markerDirPath, - long singleKey, - long value, - boolean multiKey, - int recordsBeforeFailure, - int recordsAfterFailure) { + long key, + long committedValue, + int committedRecords, + long dirtyValue, + int dirtyRecords, + long recoveryValue, + int recoveryRecords) { this.sourceId = sourceId; this.markerDirPath = markerDirPath; - this.singleKey = singleKey; - this.value = value; - this.multiKey = multiKey; - this.recordsBeforeFailure = recordsBeforeFailure; - this.recordsAfterFailure = recordsAfterFailure; + this.key = key; + this.committedValue = committedValue; + this.committedRecords = committedRecords; + this.dirtyValue = dirtyValue; + this.dirtyRecords = dirtyRecords; + this.recoveryValue = recoveryValue; + this.recoveryRecords = recoveryRecords; } - private boolean hasFailedGlobally() { - return new java.io.File(markerDirPath, sourceId).exists(); - } - - private void markFailedGlobally() { - java.io.File markerDir = new java.io.File(markerDirPath); - markerDir.mkdirs(); - try { - new java.io.File(markerDir, sourceId).createNewFile(); - } catch (Exception e) { - // Ignore + @Override + public void start() { + if (markerExists(FAILURE_MARKER_SUFFIX)) { + createMarker(RESTART_MARKER_SUFFIX); } } - @Override - public void start() {} - @Override public InputStatus pollNext(ReaderOutput output) throws Exception { - if (splits.isEmpty()) { + if (activeSplit == null) { return InputStatus.NOTHING_AVAILABLE; } - // Calculate total records based on mode - int totalRecordsBeforeFailure = - multiKey ? recordsBeforeFailure * 3 : recordsBeforeFailure; - int totalRecordsAfterFailure = multiKey ? recordsAfterFailure * 3 : recordsAfterFailure; - int totalRecords = totalRecordsBeforeFailure + totalRecordsAfterFailure; - - // Check if we should fail (one-shot) - // Use file-based marker to track failure across restarts - if (!hasFailed && !hasFailedGlobally() && totalEmitted >= totalRecordsBeforeFailure) { - hasFailed = true; - markFailedGlobally(); + int dirtyEnd = committedRecords + dirtyRecords; + int totalRecords = dirtyEnd + recoveryRecords; + long value; + if (emittedRecords < committedRecords) { + value = committedValue; + } else if (emittedRecords < dirtyEnd) { + if (!markerExists(DIRTY_RELEASE_MARKER_SUFFIX)) { + return waitForMarker(); + } + value = dirtyValue; + } else if (!markerExists(FAILURE_MARKER_SUFFIX)) { + if (!markerExists(FAILURE_TRIGGER_MARKER_SUFFIX)) { + return waitForMarker(); + } + createMarker(FAILURE_MARKER_SUFFIX); throw new RuntimeException("Injected failure for testing checkpoint recovery"); + } else if (emittedRecords < totalRecords) { + value = recoveryValue; + } else { + return waitForMarker(); } - // Check if we've emitted all records - if (totalEmitted >= totalRecords) { - // All records emitted, idle (don't return END_OF_INPUT) - // This allows checkpoints/savepoints to be taken - return InputStatus.NOTHING_AVAILABLE; - } - - // Emit a record GenericRowData row = new GenericRowData(2); - if (multiKey) { - row.setField(0, (long) (keyIndex + 1)); - keyIndex = (keyIndex + 1) % 3; - } else { - row.setField(0, singleKey); - } + row.setField(0, key); row.setField(1, value); output.collect(row); - totalEmitted++; - - // Small delay to avoid overwhelming the system - Thread.sleep(10); - + emittedRecords++; + if (markerExists(FAILURE_MARKER_SUFFIX) && emittedRecords == totalRecords) { + createMarker(RECOVERY_COMPLETE_MARKER_SUFFIX); + } return InputStatus.MORE_AVAILABLE; } + private InputStatus waitForMarker() throws InterruptedException { + Thread.sleep(MARKER_POLL_INTERVAL_MILLIS); + return InputStatus.NOTHING_AVAILABLE; + } + @Override public List snapshotState(long checkpointId) { - // Checkpoint the current state including hasFailed flag and keyIndex + if (activeSplit == null) { + return Collections.emptyList(); + } return Collections.singletonList( - new FailingCountingSplit(0, totalEmitted, hasFailed, keyIndex)); + new FailingCountingSplit(activeSplit.splitId, emittedRecords)); } @Override public CompletableFuture isAvailable() { - return availableFuture; + return activeSplit == null ? splitAvailability : AVAILABLE; } @Override - public void addSplits(List newSplits) { - // Restore state from split (assigned by Enumerator or restored from checkpoint) - for (FailingCountingSplit split : newSplits) { - this.totalEmitted = split.getEmittedRecords(); - this.hasFailed = split.hasFailed(); - this.keyIndex = split.getKeyIndex(); - splits.add(split); + public void addSplits(List splits) { + if (activeSplit != null || splits.size() != 1) { + throw new IllegalArgumentException("Expected exactly one assigned split"); } - availableFuture.complete(null); - availableFuture = new CompletableFuture<>(); + activeSplit = splits.get(0); + emittedRecords = activeSplit.emittedRecords; + splitAvailability.complete(null); } @Override - public void notifyNoMoreSplits() { - // No action needed - } + public void notifyNoMoreSplits() {} @Override public void close() {} - } - // ==================== Serializers ==================== + private boolean markerExists(String suffix) { + return new File(markerDirPath, sourceId + suffix).isFile(); + } + + private void createMarker(String suffix) { + try { + FailingCountingSource.createMarker(markerDirPath, sourceId + suffix); + } catch (IOException e) { + throw new RuntimeException("Failed to create source coordination marker", e); + } + } + } - /** - * Serializer for FailingCountingSplit. - * - *

Serialization format: splitId (4 bytes) + emittedRecords (4 bytes) + hasFailed (1 byte) + - * keyIndex (4 bytes) - */ private static class FailingCountingSplitSerializer implements SimpleVersionedSerializer { - private static final int VERSION = 2; + private static final int VERSION = 3; + private static final int SERIALIZED_LENGTH = 8; @Override public int getVersion() { @@ -562,34 +445,25 @@ public int getVersion() { @Override public byte[] serialize(FailingCountingSplit split) { - ByteBuffer buffer = ByteBuffer.allocate(13); - buffer.putInt(split.getSplitIdInt()); - buffer.putInt(split.getEmittedRecords()); - buffer.put((byte) (split.hasFailed() ? 1 : 0)); - buffer.putInt(split.getKeyIndex()); + ByteBuffer buffer = ByteBuffer.allocate(SERIALIZED_LENGTH); + buffer.putInt(split.splitId); + buffer.putInt(split.emittedRecords); return buffer.array(); } @Override - public FailingCountingSplit deserialize(int version, byte[] serialized) { + public FailingCountingSplit deserialize(int version, byte[] serialized) throws IOException { + if (version != VERSION || serialized.length != SERIALIZED_LENGTH) { + throw new IOException("Unsupported split state"); + } ByteBuffer buffer = ByteBuffer.wrap(serialized); - int splitId = buffer.getInt(); - int emittedRecords = buffer.getInt(); - boolean hasFailed = buffer.get() == 1; - int keyIndex = version >= 2 ? buffer.getInt() : 0; - return new FailingCountingSplit(splitId, emittedRecords, hasFailed, keyIndex); + return new FailingCountingSplit(buffer.getInt(), buffer.getInt()); } } - /** - * Serializer for FailingEnumeratorState. - * - *

Serialization format: recordsBeforeFailure (4 bytes) + recordsAfterFailure (4 bytes) + - * multiKey (1 byte) + splitAssigned (1 byte) - */ private static class FailingEnumeratorStateSerializer implements SimpleVersionedSerializer { - private static final int VERSION = 1; + private static final int VERSION = 2; @Override public int getVersion() { @@ -598,23 +472,32 @@ public int getVersion() { @Override public byte[] serialize(FailingEnumeratorState state) { - ByteBuffer buffer = ByteBuffer.allocate(10); - buffer.putInt(state.getRecordsBeforeFailure()); - buffer.putInt(state.getRecordsAfterFailure()); - buffer.put((byte) (state.isMultiKey() ? 1 : 0)); - buffer.put((byte) (state.isSplitAssigned() ? 1 : 0)); + if (state.pendingSplit == null) { + return new byte[] {1}; + } + ByteBuffer buffer = ByteBuffer.allocate(9); + buffer.put((byte) 0); + buffer.putInt(state.pendingSplit.splitId); + buffer.putInt(state.pendingSplit.emittedRecords); return buffer.array(); } @Override - public FailingEnumeratorState deserialize(int version, byte[] serialized) { + public FailingEnumeratorState deserialize(int version, byte[] serialized) + throws IOException { + if (version != VERSION || (serialized.length != 1 && serialized.length != 9)) { + throw new IOException("Unsupported enumerator state"); + } ByteBuffer buffer = ByteBuffer.wrap(serialized); - int recordsBeforeFailure = buffer.getInt(); - int recordsAfterFailure = buffer.getInt(); - boolean multiKey = buffer.get() == 1; - boolean splitAssigned = buffer.get() == 1; - return new FailingEnumeratorState( - recordsBeforeFailure, recordsAfterFailure, multiKey, splitAssigned); + byte assigned = buffer.get(); + if (assigned == 1 && serialized.length == 1) { + return new FailingEnumeratorState(null); + } + if (assigned == 0 && serialized.length == 9) { + return new FailingEnumeratorState( + new FailingCountingSplit(buffer.getInt(), buffer.getInt())); + } + throw new IOException("Invalid enumerator state"); } } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorITCase.java new file mode 100644 index 0000000000..ca50f507af --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorITCase.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink.undo; + +import org.apache.fluss.client.Connection; +import org.apache.fluss.client.ConnectionFactory; +import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.lookup.Lookuper; +import org.apache.fluss.client.table.Table; +import org.apache.fluss.client.table.writer.UpsertResult; +import org.apache.fluss.client.table.writer.UpsertWriter; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.sink.state.WriterState; +import org.apache.fluss.flink.sink.state.WriterStateSerializer; +import org.apache.fluss.metadata.AggFunctions; +import org.apache.fluss.metadata.MergeEngineType; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.row.InternalRow; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.Map; + +import static org.apache.fluss.testutils.DataTestUtils.row; +import static org.assertj.core.api.Assertions.assertThat; + +/** Integration test for restoring legacy markerless state through the UNDO path. */ +class UndoRecoveryOperatorITCase { + + @RegisterExtension + static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + + private static final int MAX_PARALLELISM = 4; + private static final int NUM_BUCKETS = 2; + private static final String DATABASE = FlussClusterExtension.BUILTIN_DATABASE; + + private static Connection connection; + private static Admin admin; + + @BeforeAll + static void beforeAll() { + connection = ConnectionFactory.createConnection(FLUSS_CLUSTER_EXTENSION.getClientConfig()); + admin = connection.getAdmin(); + } + + @AfterAll + static void afterAll() throws Exception { + if (admin != null) { + admin.close(); + } + if (connection != null) { + connection.close(); + } + } + + @Test + void testLegacyCheckpointWithoutActionMarkerRestoresWithUndo() throws Exception { + String suffix = Long.toUnsignedString(System.nanoTime()); + TablePath tablePath = TablePath.of(DATABASE, "legacy_markerless_undo_" + suffix); + String producerId = "legacy-markerless-undo-" + suffix; + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("value", DataTypes.BIGINT(), AggFunctions.SUM()) + .primaryKey("id") + .build(); + + admin.createTable( + tablePath, + TableDescriptor.builder() + .schema(schema) + .distributedBy(NUM_BUCKETS, "id") + .property( + ConfigOptions.TABLE_MERGE_ENGINE, + MergeEngineType.AGGREGATION) + .build(), + false) + .get(); + long tableId = admin.getTableInfo(tablePath).get().getTableId(); + FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId); + + try (Table table = connection.getTable(tablePath)) { + UpsertWriter writer = table.newUpsert().createWriter(); + Lookuper lookuper = table.newLookup().createLookuper(); + + UpsertResult committedWrite = writer.upsert(row(1, 40L)).get(); + writer.flush(); + TableBucket bucket = new TableBucket(tableId, committedWrite.getBucket().getBucket()); + Map committedBaseline = + Collections.singletonMap(bucket, committedWrite.getLogEndOffset()); + OperatorSubtaskState legacyCheckpoint = createLegacyCheckpoint(committedBaseline); + + writer.upsert(row(1, 7L)).get(); + writer.flush(); + assertThat(lookupAggregateValue(lookuper, 1)).isEqualTo(47L); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(tablePath, schema.getRowType(), producerId)) { + harness.initializeState(legacyCheckpoint); + harness.open(); + + assertThat(operator(harness).getBucketOffsets()) + .containsExactlyInAnyOrderEntriesOf(committedBaseline); + assertThat(lookupAggregateValue(lookuper, 1)).isEqualTo(40L); + } + } finally { + admin.deleteProducerOffsets(producerId).get(); + admin.dropTable(tablePath, true).get(); + } + } + + private static long lookupAggregateValue(Lookuper lookuper, int key) throws Exception { + InternalRow result = lookuper.lookup(row(key)).get().getSingletonRow(); + assertThat(result).as("aggregate row for key %s", key).isNotNull(); + return result.getLong(1); + } + + private static OperatorSubtaskState createLegacyCheckpoint(Map offsets) + throws Exception { + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>( + new LegacyCheckpointStateOperator(offsets), MAX_PARALLELISM, 1, 0)) { + harness.open(); + return harness.snapshot(1L, 1L); + } + } + + private static OneInputStreamOperatorTestHarness createHarness( + TablePath tablePath, RowType tableRowType, String producerId) throws Exception { + Configuration configuration = FLUSS_CLUSTER_EXTENSION.getClientConfig(); + UndoRecoveryOperatorFactory factory = + new UndoRecoveryOperatorFactory<>( + tablePath, + configuration, + tableRowType, + null, + NUM_BUCKETS, + false, + producerId, + RecoveryAction.UNDO, + 1L, + 1L); + return new OneInputStreamOperatorTestHarness<>(factory, MAX_PARALLELISM, 1, 0); + } + + @SuppressWarnings("unchecked") + private static UndoRecoveryOperator operator( + OneInputStreamOperatorTestHarness harness) { + return (UndoRecoveryOperator) harness.getOperator(); + } + + private static final class LegacyCheckpointStateOperator extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final long serialVersionUID = 1L; + + private final Map offsets; + + private LegacyCheckpointStateOperator(Map offsets) { + this.offsets = offsets; + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + ListState undoState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "undo_recovery_state", new WriterStateSerializer())); + undoState.add(new WriterState(offsets)); + } + + @Override + public void processElement(StreamRecord element) { + output.collect(element); + } + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorTest.java new file mode 100644 index 0000000000..8bbbee74f3 --- /dev/null +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/undo/UndoRecoveryOperatorTest.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.flink.sink.undo; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.sink.state.WriterState; +import org.apache.fluss.flink.sink.state.WriterStateSerializer; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; + +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeutils.base.StringSerializer; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link UndoRecoveryOperator}. */ +class UndoRecoveryOperatorTest { + + private static final int MAX_PARALLELISM = 4; + + private static final TableBucket BUCKET_0 = new TableBucket(1L, null, 0); + + @Test + void testNoOpNonZeroSubtaskLifecycleDoesNotUseRecoveryClient() throws Exception { + try (OneInputStreamOperatorTestHarness harness = + createHarness(RecoveryAction.NO_OP, 2, 1)) { + harness.open(); + + UndoRecoveryOperator operator = operator(harness); + operator.reportOffset(BUCKET_0, 7L); + harness.processElement(new StreamRecord<>(42)); + harness.snapshot(1L, 1L); + harness.notifyOfCompletedCheckpoint(1L); + harness.endInput(); + + assertThat(harness.extractOutputValues()).containsExactly(42); + } + } + + @Test + void testNoOpDefaultProducerDoesNotUseRecoveryClient() throws Exception { + Configuration configuration = new Configuration(); + configuration.setString(ConfigOptions.BOOTSTRAP_SERVERS.key(), "127.0.0.1:1"); + configuration.set(ConfigOptions.CLIENT_REQUEST_TIMEOUT, Duration.ofMillis(100)); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(RecoveryAction.NO_OP, 1, 0, configuration, null)) { + harness.open(); + harness.processElement(new StreamRecord<>(42)); + + assertThat(harness.extractOutputValues()).containsExactly(42); + } + } + + @Test + void testNoOpReplacesLegacyUndoStateWithActionMarker() throws Exception { + Map legacyOffsets = Collections.singletonMap(BUCKET_0, 7L); + OperatorSubtaskState legacyCheckpoint = + createCheckpoint(legacyOffsets, Collections.emptyList()); + Configuration configuration = new Configuration(); + configuration.setString(ConfigOptions.BOOTSTRAP_SERVERS.key(), "127.0.0.1:1"); + + OperatorSubtaskState noOpCheckpoint; + try (OneInputStreamOperatorTestHarness harness = + createHarness(RecoveryAction.NO_OP, 1, 0, configuration, null)) { + harness.initializeState(legacyCheckpoint); + harness.open(); + + UndoRecoveryOperator operator = operator(harness); + assertThat(operator.getBucketOffsets()).isEmpty(); + operator.reportOffset(BUCKET_0, 8L); + assertThat(operator.getBucketOffsets()).isEmpty(); + noOpCheckpoint = harness.snapshot(2L, 2L); + } + + StateProbeOperator probe = new StateProbeOperator(); + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>(probe, MAX_PARALLELISM, 1, 0)) { + harness.initializeState(noOpCheckpoint); + harness.open(); + + assertThat(probe.writerStates).isEmpty(); + assertThat(probe.recoveryActionMarkers).containsExactly(RecoveryAction.NO_OP.name()); + } + } + + @Test + void testNoOpCheckpointCannotRestoreWithUndo() throws Exception { + OperatorSubtaskState noOpCheckpoint = + createCheckpoint( + Collections.emptyMap(), + Collections.singletonList(RecoveryAction.NO_OP.name())); + try (OneInputStreamOperatorTestHarness undo = + createHarness(RecoveryAction.UNDO, 1, 0)) { + assertThatThrownBy(() -> undo.initializeState(noOpCheckpoint)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("NO_OP") + .hasMessageContaining("UNDO"); + } + } + + private static OperatorSubtaskState createCheckpoint( + Map offsets, List recoveryActionMarkers) throws Exception { + try (OneInputStreamOperatorTestHarness harness = + new OneInputStreamOperatorTestHarness<>( + new CheckpointStateOperator(offsets, recoveryActionMarkers), + MAX_PARALLELISM, + 1, + 0)) { + harness.open(); + return harness.snapshot(1L, 1L); + } + } + + private static OneInputStreamOperatorTestHarness createHarness( + RecoveryAction action, int parallelism, int subtaskIndex) throws Exception { + Configuration configuration = new Configuration(); + configuration.setString(ConfigOptions.BOOTSTRAP_SERVERS.key(), "127.0.0.1:1"); + return createHarness(action, parallelism, subtaskIndex, configuration); + } + + private static OneInputStreamOperatorTestHarness createHarness( + RecoveryAction action, int parallelism, int subtaskIndex, Configuration configuration) + throws Exception { + return createHarness(action, parallelism, subtaskIndex, configuration, "test-producer"); + } + + private static OneInputStreamOperatorTestHarness createHarness( + RecoveryAction action, + int parallelism, + int subtaskIndex, + Configuration configuration, + String producerId) + throws Exception { + UndoRecoveryOperatorFactory factory = + new UndoRecoveryOperatorFactory<>( + TablePath.of("db", "table"), + configuration, + DATA1_ROW_TYPE, + null, + 2, + false, + producerId, + action, + 1L, + 1L); + return new OneInputStreamOperatorTestHarness<>( + factory, MAX_PARALLELISM, parallelism, subtaskIndex); + } + + @SuppressWarnings("unchecked") + private static UndoRecoveryOperator operator( + OneInputStreamOperatorTestHarness harness) { + return (UndoRecoveryOperator) harness.getOperator(); + } + + private static final class CheckpointStateOperator extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final long serialVersionUID = 1L; + + private final Map offsets; + private final List recoveryActionMarkers; + + private CheckpointStateOperator( + Map offsets, List recoveryActionMarkers) { + this.offsets = offsets; + this.recoveryActionMarkers = recoveryActionMarkers; + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + ListState undoState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "undo_recovery_state", new WriterStateSerializer())); + if (!offsets.isEmpty()) { + undoState.add(new WriterState(offsets)); + } + + ListState recoveryActionState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "recovery_action_state", StringSerializer.INSTANCE)); + for (String recoveryActionMarker : recoveryActionMarkers) { + recoveryActionState.add(recoveryActionMarker); + } + } + + @Override + public void processElement(StreamRecord element) { + output.collect(element); + } + } + + private static final class StateProbeOperator extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final long serialVersionUID = 1L; + + private final List writerStates = new ArrayList<>(); + private final List recoveryActionMarkers = new ArrayList<>(); + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + ListState undoState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "undo_recovery_state", new WriterStateSerializer())); + for (WriterState writerState : undoState.get()) { + writerStates.add(writerState); + } + + ListState recoveryActionState = + context.getOperatorStateStore() + .getUnionListState( + new ListStateDescriptor<>( + "recovery_action_state", StringSerializer.INSTANCE)); + for (String recoveryActionMarker : recoveryActionState.get()) { + recoveryActionMarkers.add(recoveryActionMarker); + } + } + + @Override + public void processElement(StreamRecord element) { + output.collect(element); + } + } +} diff --git a/website/docs/engine-flink/writes.md b/website/docs/engine-flink/writes.md index 354aa97475..74ab5a95f2 100644 --- a/website/docs/engine-flink/writes.md +++ b/website/docs/engine-flink/writes.md @@ -87,6 +87,41 @@ INSERT INTO pk_table (shop_id, user_id, num_orders) SELECT shop_id, user_id, num_orders FROM source; ``` +## Failover Recovery for Aggregation Tables + +For tables that use the aggregation merge engine, the Flink connector automatically selects the +failover recovery behavior from the aggregation functions and the records that the sink can emit. +No additional connector option is required. + +The connector skips undo recovery when every column written by the job uses an idempotent +aggregation function and deletes cannot change the materialized state. Applying the same +aggregation input again does not change the materialized result for these functions: + +- `max` and `min` +- `first_value` and `first_value_ignore_nulls` +- `last_value` and `last_value_ignore_nulls` +- `bool_and` and `bool_or` +- `rbm32` and `rbm64` + +Undo recovery remains enabled for non-idempotent functions such as `sum`, `product`, `listagg`, +and `string_agg`. It is also retained when the input can contain an effective delete or when the +connector cannot prove that skipping recovery is safe. For partial updates, only the columns +written by the job participate in this decision. + +When undo recovery is skipped, the connector does not track bucket offsets or apply data undo. +When `sink.producer-id` is configured, subtask 0 removes stale producer offsets for that ID before +records are processed; the configured ID must not be shared by concurrently running sinks. The +default Flink job ID is not used for this cleanup because all sinks in a job share it; a fresh job +submission naturally receives a new ID. Other subtasks do not connect for recovery. + +:::note Checkpoint compatibility +Markerless checkpoints from older connector versions use the recovery action selected by the +current job. A checkpoint with undo recovery skipped clears old undo state and cannot later be +restored by a job revision that requires undo recovery, because it contains no bucket-offset +baseline. Keep the same recovery-safe write semantics when restoring such a checkpoint, or start +the changed job without restoring it. +::: + ## DELETE FROM Fluss supports deleting data for primary-key tables in batch mode via `DELETE FROM` statement. Currently, only single data deletions based on the primary key are supported. @@ -113,4 +148,4 @@ SET execution.runtime-mode = batch; ```sql title="Flink SQL" -- The condition must include all primary key equality conditions. UPDATE pk_table SET total_amount = 2 WHERE shop_id = 10000 AND user_id = 123456; -``` \ No newline at end of file +```