From a32fdec6afaf50df45bfeb922f025d7057cce853 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 11 Aug 2026 10:20:50 +0800 Subject: [PATCH 01/11] [fix](routineload) Persist the current load definition ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Routine Load image recovery reparsed the immutable original CREATE statement, so CREATE semantics changed by ALTER were not represented in the image. Persist the current load definition, retain the original statement as the legacy-image fallback, journal altered load clauses, and validate failure-prone Kafka and Kinesis changes before mutating runtime state. ### Release note Routine Load jobs now recover the current effective load definition after ALTER. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest - Behavior changed: Yes. Image recovery uses the current effective Routine Load definition while legacy images continue to use origStmt. - Does this need documentation: No --- .../org/apache/doris/analysis/Separator.java | 3 + .../apache/doris/load/RoutineLoadDesc.java | 10 + .../routineload/RoutineLoadDefinition.java | 128 ++++++++++++ .../load/routineload/RoutineLoadJob.java | 137 ++++++++++--- .../load/routineload/RoutineLoadManager.java | 1 - .../kafka/KafkaRoutineLoadJob.java | 186 +++++++++++------ .../kinesis/KinesisRoutineLoadJob.java | 187 +++++++++++------ .../AlterRoutineLoadJobOperationLog.java | 13 ++ .../routineload/KafkaRoutineLoadJobTest.java | 188 ++++++++++++++++++ .../KinesisRoutineLoadJobTest.java | 48 +++++ .../AlterRoutineLoadOperationLogTest.java | 24 ++- 11 files changed, 773 insertions(+), 152 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java index 67515eaca5c79f..7da2e092a212ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java @@ -20,13 +20,16 @@ import org.apache.doris.common.AnalysisException; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import java.io.StringWriter; public class Separator { private static final String HEX_STRING = "0123456789ABCDEF"; + @SerializedName("os") private final String oriSeparator; + @SerializedName("s") private String separator; public Separator(String separator) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java index 2c1ede0d13a352..28a429960d2b71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java @@ -26,19 +26,29 @@ import org.apache.doris.load.loadv2.LoadTask; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import java.util.List; public class RoutineLoadDesc { + @SerializedName("cs") private final Separator columnSeparator; + @SerializedName("ld") private final Separator lineDelimiter; + @SerializedName("cols") private final List columnsInfo; + @SerializedName("pf") private final Expr precedingFilter; + @SerializedName("f") private final Expr filter; + @SerializedName("dc") private final Expr deleteCondition; + @SerializedName("mt") private LoadTask.MergeType mergeType; // nullable + @SerializedName("pn") private final PartitionNamesInfo partitionNamesInfo; + @SerializedName("sc") private final String sequenceColName; public RoutineLoadDesc(Separator columnSeparator, Separator lineDelimiter, List columnsInfo, diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java new file mode 100644 index 00000000000000..bcfb59c48d90f4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.load.routineload; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.common.UserException; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.nereids.load.NereidsLoadUtils; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; +import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnDesc; +import org.apache.doris.nereids.trees.plans.commands.load.LoadDeleteOnClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadPartitionNames; +import org.apache.doris.nereids.trees.plans.commands.load.LoadPrecedingFilterClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; +import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; +import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause; +import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause; +import org.apache.doris.nereids.trees.expressions.Expression; + +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Snapshot of the current CREATE ROUTINE LOAD semantics. + */ +public class RoutineLoadDefinition { + @SerializedName("desc") + private RoutineLoadDesc routineLoadDesc; + @SerializedName("jp") + private Map jobProperties = Maps.newHashMap(); + @SerializedName("dsp") + private Map dataSourceProperties = Maps.newHashMap(); + + public RoutineLoadDefinition(RoutineLoadDesc routineLoadDesc, + Map jobProperties, Map dataSourceProperties) { + this.routineLoadDesc = routineLoadDesc; + this.jobProperties.putAll(jobProperties); + this.dataSourceProperties.putAll(dataSourceProperties); + } + + public RoutineLoadDesc getRoutineLoadDesc() { + return routineLoadDesc; + } + + public Map getDataSourceProperties() { + return dataSourceProperties; + } + + public CreateRoutineLoadInfo toCreateInfo(String dbName, String jobName, String tableName, + LoadDataSourceType dataSourceType, String comment) throws UserException { + LoadTask.MergeType mergeType = routineLoadDesc == null + ? LoadTask.MergeType.APPEND : routineLoadDesc.getMergeType(); + return new CreateRoutineLoadInfo(new LabelNameInfo(dbName, jobName), tableName, + toLoadPropertyMap(routineLoadDesc), Maps.newHashMap(jobProperties), dataSourceType.name(), + Maps.newHashMap(dataSourceProperties), mergeType, comment); + } + + private static Map toLoadPropertyMap(RoutineLoadDesc routineLoadDesc) throws UserException { + Map loadProperties = Maps.newHashMap(); + if (routineLoadDesc == null) { + return loadProperties; + } + if (routineLoadDesc.getColumnSeparator() != null) { + put(loadProperties, new LoadSeparator(routineLoadDesc.getColumnSeparator().getOriSeparator())); + } + if (routineLoadDesc.getColumnsInfo() != null) { + List columns = new ArrayList<>(); + for (ImportColumnDesc column : routineLoadDesc.getColumnsInfo()) { + Expression expression = column.getExpr() == null ? null : parseExpression(column.getExpr()); + columns.add(new LoadColumnDesc(column.getColumnName(), expression)); + } + put(loadProperties, new LoadColumnClause(columns)); + } + if (routineLoadDesc.getPrecedingFilter() != null) { + put(loadProperties, new LoadPrecedingFilterClause( + parseExpression(routineLoadDesc.getPrecedingFilter()))); + } + if (routineLoadDesc.getFilter() != null) { + put(loadProperties, new LoadWhereClause(parseExpression(routineLoadDesc.getFilter()))); + } + if (routineLoadDesc.getPartitionNamesInfo() != null) { + put(loadProperties, new LoadPartitionNames( + routineLoadDesc.getPartitionNamesInfo().isTemp(), + routineLoadDesc.getPartitionNamesInfo().getPartitionNames())); + } + if (routineLoadDesc.getDeleteCondition() != null) { + put(loadProperties, new LoadDeleteOnClause(parseExpression(routineLoadDesc.getDeleteCondition()))); + } + if (routineLoadDesc.hasSequenceCol()) { + put(loadProperties, new LoadSequenceClause(routineLoadDesc.getSequenceColName())); + } + return loadProperties; + } + + private static Expression parseExpression(Expr expression) throws UserException { + String sql = expression.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + return NereidsLoadUtils.parseExpressionSeq(sql).get(0); + } + + private static void put(Map loadProperties, LoadProperty loadProperty) { + loadProperties.put(loadProperty.getClass().getName(), loadProperty); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 9873368f405114..4229aca376353e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -258,8 +259,11 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // this is the origin stmt of CreateRoutineLoadStmt, we use it to persist the RoutineLoadJob, - // because we can not serialize the Expressions contained in job. + // Canonical current CREATE semantics. CREATE and ALTER must keep this snapshot current. + @SerializedName("ld") + protected RoutineLoadDefinition loadDefinition; + + // Legacy recovery input for images written before loadDefinition was persisted. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -470,6 +474,62 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } + protected RoutineLoadDesc getLoadDefinitionRoutineLoadDesc() { + List columnsInfo = columnDescs == null ? null : columnDescs.descs; + return new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, + partitionNamesInfo, deleteCondition, mergeType, sequenceCol); + } + + protected void initializeLoadDefinition(CreateRoutineLoadInfo info) { + Map originalDataSourceProperties = + info.getDataSourceProperties().getOriginalDataSourceProperties(); + Map dataSourceProperties = originalDataSourceProperties == null + ? Maps.newHashMap() : Maps.newHashMap(originalDataSourceProperties); + updateLoadDefinitionDataSourceProperties(dataSourceProperties); + loadDefinition = new RoutineLoadDefinition( + getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); + } + + protected void updateLoadDefinition(AbstractDataSourceProperties changedDataSourceProperties) { + Map dataSourceProperties = loadDefinition == null + ? Maps.newHashMap() : Maps.newHashMap(loadDefinition.getDataSourceProperties()); + if (changedDataSourceProperties != null + && changedDataSourceProperties.getOriginalDataSourceProperties() != null) { + dataSourceProperties.putAll(changedDataSourceProperties.getOriginalDataSourceProperties()); + } + updateLoadDefinitionDataSourceProperties(dataSourceProperties); + loadDefinition = new RoutineLoadDefinition( + getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); + } + + protected Map snapshotLoadDefinitionJobProperties() { + Map currentJobProperties = Maps.newHashMap(jobProperties); + currentJobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, + String.valueOf(desireTaskConcurrentNum)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, String.valueOf(maxErrorNum)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, String.valueOf(maxFilterRatio)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_INTERVAL_SEC_PROPERTY, + String.valueOf(maxBatchIntervalS)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_ROWS_PROPERTY, String.valueOf(maxBatchRows)); + currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY, String.valueOf(maxBatchSizeBytes)); + currentJobProperties.put(CreateRoutineLoadInfo.EXEC_MEM_LIMIT_PROPERTY, String.valueOf(execMemLimit)); + currentJobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, + String.valueOf(sendBatchParallelism)); + currentJobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, + String.valueOf(loadToSingleTablet)); + currentJobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); + currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); + if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + currentJobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + } else { + currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, + partialUpdateNewKeyPolicy.name()); + } + return currentJobProperties; + } + + protected abstract void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties); + @Override public long getId() { return id; @@ -2009,45 +2069,67 @@ public void gsonPostProcess() throws IOException { ctx.getState().reset(); try { ctx.setThreadLocalInfo(); - NereidsParser nereidsParser = new NereidsParser(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( - origStmt.originStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - // If tableId is set, resolve the current table name by ID so that - // table rename / SWAP TABLE won't cause replay to fail with stale name in origStmt. - if (!isMultiTable && tableId != 0) { - try { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); - if (db != null) { - db.getTable(tableId).ifPresent( - table -> createRoutineLoadInfo.setTableName(table.getName())); - } - } catch (Exception ignored) { - // fall through; let validate() surface the real error - } + if (loadDefinition == null) { + restoreLegacyDefinition(ctx); + } else { + restoreLoadDefinition(ctx); } - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); } finally { ctx.cleanup(); } } catch (Exception e) { this.state = JobState.CANCELLED; - LOG.warn("error happens when parsing create routine load stmt: " + origStmt.originStmt, e); + LOG.warn("error happens when restoring routine load definition", e); } if (userIdentity != null) { userIdentity.setIsAnalyzed(); } } + private void restoreLegacyDefinition(ConnectContext ctx) throws UserException { + NereidsParser nereidsParser = new NereidsParser(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( + origStmt.originStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + if (!isMultiTable) { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); + createRoutineLoadInfo.setTableName(db.getTable(tableId).get().getName()); + } + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); + } + + private void restoreLoadDefinition(ConnectContext ctx) throws UserException { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); + String tableName = isMultiTable ? null : db.getTable(tableId).get().getName(); + CreateRoutineLoadInfo createRoutineLoadInfo = loadDefinition.toCreateInfo( + db.getFullName(), name, tableName, dataSourceType, comment); + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(loadDefinition.getRoutineLoadDesc()); + } + public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; public abstract void replayModifyProperties(AlterRoutineLoadJobOperationLog log); public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; - // for ALTER ROUTINE LOAD - protected void modifyCommonJobProperties(Map jobProperties) throws UserException { + protected TUniqueKeyUpdateMode validateCommonJobProperties(Map jobProperties) + throws UserException { + if (!jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + return null; + } + TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } + return newMode; + } + + // for ALTER ROUTINE LOAD. All failure-prone validation must be completed before calling this method. + protected void modifyCommonJobProperties(Map jobProperties, + TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode) { if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); @@ -2080,13 +2162,8 @@ protected void modifyCommonJobProperties(Map jobProperties) thro } if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); - // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); - } - this.uniqueKeyUpdateMode = newMode; + jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + this.uniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index df5615016bfa54..9bc9b93abd8c98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -943,7 +943,6 @@ public void alterRoutineLoadJob(AlterRoutineLoadCommand command) throws UserExce + command.getDataSourceProperties().getDataSourceType()); } job.modifyProperties(command); - job.setRoutineLoadDesc(command.getRoutineLoadDesc()); } public void replayAlterRoutineLoadJob(AlterRoutineLoadJobOperationLog log) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 885021440351d7..a5d1dd5f0e7ef7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -61,6 +61,7 @@ import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -216,19 +217,25 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - if (rebuild) { - convertedCustomProperties.clear(); - } + Pair, String> convertedProperties = buildConvertedCustomProperties( + customProperties, kafkaDefaultOffSet); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(convertedProperties.first); + kafkaDefaultOffSet = convertedProperties.second; + } - SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); - for (Map.Entry entry : customProperties.entrySet()) { + private Pair, String> buildConvertedCustomProperties( + Map sourceProperties, String currentDefaultOffset) throws DdlException { + Map convertedProperties = Maps.newHashMap(); + for (Map.Entry entry : sourceProperties.entrySet()) { if (entry.getValue().startsWith("FILE:")) { // convert FILE:file_name -> FILE:file_id:md5 String file = entry.getValue().substring(entry.getValue().indexOf(":") + 1); + SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); SmallFile smallFile = smallFileMgr.getSmallFile(dbId, KAFKA_FILE_CATALOG, file, true); - convertedCustomProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); + convertedProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); } else { - convertedCustomProperties.put(entry.getKey(), entry.getValue()); + convertedProperties.put(entry.getKey(), entry.getValue()); } } @@ -237,14 +244,14 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - kafkaDefaultOffSet = convertedCustomProperties + String convertedDefaultOffset = currentDefaultOffset; + if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + convertedDefaultOffset = convertedProperties .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - return; - } - if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { - kafkaDefaultOffSet = convertedCustomProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + } else if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { + convertedDefaultOffset = convertedProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); } + return Pair.of(convertedProperties, convertedDefaultOffset); } @Override @@ -590,6 +597,7 @@ public static KafkaRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, Con kafkaRoutineLoadJob.setOptional(info); kafkaRoutineLoadJob.checkCustomProperties(); kafkaRoutineLoadJob.checkCustomPartition(); + kafkaRoutineLoadJob.initializeLoadDefinition(info); return kafkaRoutineLoadJob; } @@ -775,6 +783,21 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } + @Override + protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { + dataSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), brokerList); + dataSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), topic); + dataSourceProperties.remove(KafkaConfiguration.KAFKA_OFFSETS.getName()); + if (customKafkaPartitions.isEmpty()) { + dataSourceProperties.remove(KafkaConfiguration.KAFKA_PARTITIONS.getName()); + } else { + dataSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), + Joiner.on(",").join(customKafkaPartitions)); + } + customProperties.forEach((key, value) -> dataSourceProperties.put( + key.startsWith("aws.") ? key : "property." + key, value)); + } + @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); @@ -791,9 +814,11 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + setRoutineLoadDesc(command.getRoutineLoadDesc()); + updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -820,66 +845,89 @@ private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throw private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { - if (null != dataSourceProperties) { - List> kafkaPartitionOffsets = Lists.newArrayList(); - Map customKafkaProperties = Maps.newHashMap(); + PreparedKafkaAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); + applyAlter(jobProperties, dataSourceProperties, preparedAlter); + if (LOG.isDebugEnabled()) { + LOG.debug("modify the properties of kafka routine load job: {}, jobProperties: {}, " + + "datasource properties: {}", + this.id, jobProperties, dataSourceProperties); + } + } - if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - customKafkaProperties = dataSourceProperties.getCustomKafkaProperties(); - } + private PreparedKafkaAlter prepareAlter(Map jobProperties, + KafkaDataSourceProperties dataSourceProperties) throws UserException { + TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); + List> kafkaPartitionOffsets = Lists.newArrayList(); + Map alteredCustomProperties = Maps.newHashMap(); + if (dataSourceProperties != null + && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); + } - // convertCustomProperties and check partitions before reset progress to make modify operation atomic - if (!customKafkaProperties.isEmpty()) { - this.customProperties.putAll(customKafkaProperties); - convertCustomProperties(true); - } + Map stagedCustomProperties = null; + Map stagedConvertedCustomProperties = null; + String stagedKafkaDefaultOffset = kafkaDefaultOffSet; + if (!alteredCustomProperties.isEmpty()) { + stagedCustomProperties = Maps.newHashMap(customProperties); + stagedCustomProperties.putAll(alteredCustomProperties); + Pair, String> convertedProperties = buildConvertedCustomProperties( + stagedCustomProperties, stagedKafkaDefaultOffset); + stagedConvertedCustomProperties = convertedProperties.first; + stagedKafkaDefaultOffset = convertedProperties.second; + } + if (!kafkaPartitionOffsets.isEmpty()) { + ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); + } + if (dataSourceProperties != null && Config.isCloudMode()) { + Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() + .setRequestIp(FrontendOptions.getLocalHostAddressCached()); + builder.setCloudUniqueId(Config.cloud_unique_id); + builder.setDbId(dbId); + builder.setJobId(id); if (!kafkaPartitionOffsets.isEmpty()) { - ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); - } - - if (Config.isCloudMode()) { - Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() - .setRequestIp(FrontendOptions.getLocalHostAddressCached()); - builder.setCloudUniqueId(Config.cloud_unique_id); - builder.setDbId(dbId); - builder.setJobId(id); - if (!kafkaPartitionOffsets.isEmpty()) { - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is - // this value will be incremented - // when pulling MS persistent progress data and updating memory - // in routineLoadJob.updateCloudProgress(). - partitionOffsetMap.put(pair.first, pair.second - 1); - } - builder.putAllPartitionToOffset(partitionOffsetMap); + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is + // this value will be incremented when pulling MS persistent progress data and updating memory + // in routineLoadJob.updateCloudProgress(). + partitionOffsetMap.put(pair.first, pair.second - 1); } - resetCloudProgress(builder); + builder.putAllPartitionToOffset(partitionOffsetMap); } + resetCloudProgress(builder); + } + return new PreparedKafkaAlter(validatedUniqueKeyUpdateMode, kafkaPartitionOffsets, + stagedCustomProperties, stagedConvertedCustomProperties, stagedKafkaDefaultOffset); + } + private void applyAlter(Map jobProperties, KafkaDataSourceProperties dataSourceProperties, + PreparedKafkaAlter preparedAlter) { + if (dataSourceProperties != null) { + if (preparedAlter.stagedCustomProperties != null) { + customProperties.clear(); + customProperties.putAll(preparedAlter.stagedCustomProperties); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); + kafkaDefaultOffSet = preparedAlter.stagedKafkaDefaultOffset; + } // It is necessary to reset the Kafka progress cache if topic change, // and should reset cache before modifying partition offset. if (!Strings.isNullOrEmpty(dataSourceProperties.getTopic())) { - this.topic = dataSourceProperties.getTopic(); - this.progress = new KafkaProgress(); + topic = dataSourceProperties.getTopic(); + progress = new KafkaProgress(); } - - // modify partition offset - if (!kafkaPartitionOffsets.isEmpty()) { - // we can only modify the partition that is being consumed - ((KafkaProgress) progress).modifyOffset(kafkaPartitionOffsets); + if (!preparedAlter.kafkaPartitionOffsets.isEmpty()) { + ((KafkaProgress) progress).modifyOffset(preparedAlter.kafkaPartitionOffsets); } - - // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { - this.brokerList = dataSourceProperties.getBrokerList(); + brokerList = dataSourceProperties.getBrokerList(); } } if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties); + modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); this.jobProperties.putAll(copiedJobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); @@ -893,8 +941,26 @@ private void modifyPropertiesInternal(Map jobProperties, } } } - LOG.info("modify the properties of kafka routine load job: {}, jobProperties: {}, datasource properties: {}", - this.id, jobProperties, dataSourceProperties); + } + + private static class PreparedKafkaAlter { + private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; + private final List> kafkaPartitionOffsets; + private final Map stagedCustomProperties; + private final Map stagedConvertedCustomProperties; + private final String stagedKafkaDefaultOffset; + + private PreparedKafkaAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, + List> kafkaPartitionOffsets, + Map stagedCustomProperties, + Map stagedConvertedCustomProperties, + String stagedKafkaDefaultOffset) { + this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; + this.kafkaPartitionOffsets = kafkaPartitionOffsets; + this.stagedCustomProperties = stagedCustomProperties; + this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; + this.stagedKafkaDefaultOffset = stagedKafkaDefaultOffset; + } } private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { @@ -920,6 +986,8 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); + setRoutineLoadDesc(log.getRoutineLoadDesc()); + updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 7cebc3f5165b49..5ca8355d851c50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -52,6 +52,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -186,19 +187,20 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - if (rebuild) { - convertedCustomProperties.clear(); - } - - for (Map.Entry entry : customProperties.entrySet()) { - convertedCustomProperties.put(entry.getKey(), entry.getValue()); - } + Pair, String> convertedProperties = buildConvertedCustomProperties( + customProperties, kinesisDefaultPosition); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(convertedProperties.first); + kinesisDefaultPosition = convertedProperties.second; + } - // Handle default position - if (convertedCustomProperties.containsKey("kinesis_default_pos")) { - kinesisDefaultPosition = convertedCustomProperties.get("kinesis_default_pos"); - // Keep it in convertedCustomProperties so BE can use it - } + private Pair, String> buildConvertedCustomProperties( + Map sourceProperties, String currentDefaultPosition) { + Map convertedProperties = Maps.newHashMap(sourceProperties); + String convertedDefaultPosition = convertedProperties.getOrDefault( + "kinesis_default_pos", currentDefaultPosition); + // Keep kinesis_default_pos in convertedProperties so BE can use it. + return Pair.of(convertedProperties, convertedDefaultPosition); } private String convertedDefaultPosition() { @@ -532,6 +534,7 @@ public static KinesisRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, C kinesisRoutineLoadJob.setOptional(info); kinesisRoutineLoadJob.checkCustomProperties(); + kinesisRoutineLoadJob.initializeLoadDefinition(info); return kinesisRoutineLoadJob; } @@ -660,6 +663,27 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } + @Override + protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { + dataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), region); + dataSourceProperties.put(KinesisConfiguration.KINESIS_STREAM.getName(), stream); + if (endpoint == null) { + dataSourceProperties.remove(KinesisConfiguration.KINESIS_ENDPOINT.getName()); + dataSourceProperties.remove("kinesis_endpoint"); + } else { + dataSourceProperties.put(KinesisConfiguration.KINESIS_ENDPOINT.getName(), endpoint); + } + dataSourceProperties.remove(KinesisConfiguration.KINESIS_POSITIONS.getName()); + if (customKinesisShards.isEmpty()) { + dataSourceProperties.remove(KinesisConfiguration.KINESIS_SHARDS.getName()); + } else { + dataSourceProperties.put(KinesisConfiguration.KINESIS_SHARDS.getName(), + Joiner.on(",").join(customKinesisShards)); + } + customProperties.forEach((key, value) -> dataSourceProperties.put( + key.startsWith("aws.") ? key : "property." + key, value)); + } + private Map getMaskedCustomProperties(String keyPrefix) { Map maskedProperties = new HashMap<>(); customProperties.forEach((key, value) -> { @@ -687,9 +711,11 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti } modifyPropertiesInternal(jobProperties, dataSourceProperties); + setRoutineLoadDesc(command.getRoutineLoadDesc()); + updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties); + jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -699,68 +725,85 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti private void modifyPropertiesInternal(Map jobProperties, KinesisDataSourceProperties dataSourceProperties) throws UserException { - if (dataSourceProperties != null) { - List> shardPositions = Lists.newArrayList(); - Map customKinesisProperties = Maps.newHashMap(); - boolean resetProgress = false; - boolean hasExplicitShardPositions = false; - - if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - shardPositions = dataSourceProperties.getKinesisShardPositions(); - customKinesisProperties = dataSourceProperties.getCustomKinesisProperties(); - hasExplicitShardPositions = !shardPositions.isEmpty(); - } + PreparedKinesisAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); + applyAlter(jobProperties, dataSourceProperties, preparedAlter); + LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", + this.id, jobProperties, dataSourceProperties); + } - // Update custom properties - if (!customKinesisProperties.isEmpty()) { - this.customProperties.putAll(customKinesisProperties); - convertCustomProperties(true); - } + private PreparedKinesisAlter prepareAlter(Map jobProperties, + KinesisDataSourceProperties dataSourceProperties) throws UserException { + TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); + List> shardPositions = Lists.newArrayList(); + Map alteredCustomProperties = Maps.newHashMap(); + if (dataSourceProperties != null + && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + shardPositions = dataSourceProperties.getKinesisShardPositions(); + alteredCustomProperties = dataSourceProperties.getCustomKinesisProperties(); + } - // Modify stream if provided + Map stagedCustomProperties = null; + Map stagedConvertedCustomProperties = null; + String stagedDefaultPosition = kinesisDefaultPosition; + if (!alteredCustomProperties.isEmpty()) { + stagedCustomProperties = Maps.newHashMap(customProperties); + stagedCustomProperties.putAll(alteredCustomProperties); + Pair, String> convertedProperties = buildConvertedCustomProperties( + stagedCustomProperties, stagedDefaultPosition); + stagedConvertedCustomProperties = convertedProperties.first; + stagedDefaultPosition = convertedProperties.second; + } + + boolean resetProgress = dataSourceProperties != null + && !Strings.isNullOrEmpty(dataSourceProperties.getStream()); + if (!shardPositions.isEmpty() && !resetProgress) { + ((KinesisProgress) progress).checkShards(shardPositions); + } + return new PreparedKinesisAlter(validatedUniqueKeyUpdateMode, shardPositions, + stagedCustomProperties, stagedConvertedCustomProperties, stagedDefaultPosition, resetProgress); + } + + private void applyAlter(Map jobProperties, KinesisDataSourceProperties dataSourceProperties, + PreparedKinesisAlter preparedAlter) { + if (dataSourceProperties != null) { + if (preparedAlter.stagedCustomProperties != null) { + customProperties.clear(); + customProperties.putAll(preparedAlter.stagedCustomProperties); + convertedCustomProperties.clear(); + convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); + kinesisDefaultPosition = preparedAlter.stagedDefaultPosition; + } if (!Strings.isNullOrEmpty(dataSourceProperties.getStream())) { - this.stream = dataSourceProperties.getStream(); - resetProgress = true; + stream = dataSourceProperties.getStream(); } - - // Modify region if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getRegion())) { - this.region = dataSourceProperties.getRegion(); + region = dataSourceProperties.getRegion(); } - - // Modify endpoint if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getEndpoint())) { - this.endpoint = dataSourceProperties.getEndpoint(); + endpoint = dataSourceProperties.getEndpoint(); } - - if (resetProgress) { - this.progress = new KinesisProgress(); - this.openKinesisShards.clear(); - this.closedKinesisShards.clear(); - this.cachedShardWithMillsBehindLatest.clear(); + if (preparedAlter.resetProgress) { + progress = new KinesisProgress(); + openKinesisShards.clear(); + closedKinesisShards.clear(); + cachedShardWithMillsBehindLatest.clear(); } - - if (hasExplicitShardPositions) { - this.customKinesisShards.clear(); - for (Pair shardPosition : shardPositions) { - this.customKinesisShards.add(shardPosition.first); + if (!preparedAlter.shardPositions.isEmpty()) { + customKinesisShards.clear(); + for (Pair shardPosition : preparedAlter.shardPositions) { + customKinesisShards.add(shardPosition.first); } - } else if (resetProgress) { + } else if (preparedAlter.resetProgress) { // Stream change without explicit shards should fall back to dynamic shard discovery. - this.customKinesisShards.clear(); + customKinesisShards.clear(); } - - if (!shardPositions.isEmpty()) { - if (!resetProgress) { - ((KinesisProgress) progress).checkShards(shardPositions); - } - ((KinesisProgress) progress).modifyPosition(shardPositions); + if (!preparedAlter.shardPositions.isEmpty()) { + ((KinesisProgress) progress).modifyPosition(preparedAlter.shardPositions); } } - if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties); + modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); this.jobProperties.putAll(copiedJobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); @@ -774,8 +817,28 @@ private void modifyPropertiesInternal(Map jobProperties, } } } - LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", - this.id, jobProperties, dataSourceProperties); + } + + private static class PreparedKinesisAlter { + private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; + private final List> shardPositions; + private final Map stagedCustomProperties; + private final Map stagedConvertedCustomProperties; + private final String stagedDefaultPosition; + private final boolean resetProgress; + + private PreparedKinesisAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, + List> shardPositions, + Map stagedCustomProperties, + Map stagedConvertedCustomProperties, + String stagedDefaultPosition, boolean resetProgress) { + this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; + this.shardPositions = shardPositions; + this.stagedCustomProperties = stagedCustomProperties; + this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; + this.stagedDefaultPosition = stagedDefaultPosition; + this.resetProgress = resetProgress; + } } @Override @@ -783,6 +846,8 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); + setRoutineLoadDesc(log.getRoutineLoadDesc()); + updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index 4729882f7927fb..9d8064943c566a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -19,6 +19,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.persist.gson.GsonUtils; @@ -37,12 +38,20 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private Map jobProperties; @SerializedName(value = "dataSourceProperties") private AbstractDataSourceProperties dataSourceProperties; + @SerializedName(value = "routineLoadDesc") + private RoutineLoadDesc routineLoadDesc; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { + this(jobId, jobProperties, dataSourceProperties, null); + } + + public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, + AbstractDataSourceProperties dataSourceProperties, RoutineLoadDesc routineLoadDesc) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; + this.routineLoadDesc = routineLoadDesc; } public long getJobId() { @@ -57,6 +66,10 @@ public AbstractDataSourceProperties getDataSourceProperties() { return dataSourceProperties; } + public RoutineLoadDesc getRoutineLoadDesc() { + return routineLoadDesc; + } + public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterRoutineLoadJobOperationLog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 7f0c8588372403..c3cfad1dc1ff18 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -30,6 +30,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.load.RoutineLoadDesc; @@ -40,11 +41,15 @@ import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; import org.apache.doris.load.routineload.kafka.KafkaTaskInfo; import org.apache.doris.mysql.privilege.MockedAuth; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRoutineLoadTask; @@ -58,14 +63,20 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class KafkaRoutineLoadJobTest { @@ -272,6 +283,183 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro Assert.assertTrue(otherMsg.contains("some records may be in uncommitted transactions")); } + @Test + public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job_atomic", 1L, + 1L, "127.0.0.1:9020", "topic-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map originalCustomProperties = Maps.newHashMap(); + originalCustomProperties.put("client.id", "old-client"); + Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); + Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); + Map originalProgress = Maps.newHashMap(); + originalProgress.put(0, 10L); + Deencapsulation.setField(routineLoadJob, "progress", new KafkaProgress(originalProgress)); + routineLoadJob.updateLoadDefinition(null); + Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + + Map originalDataSourceProperties = Maps.newHashMap(); + originalDataSourceProperties.put("property.client.id", "new-client"); + KafkaDataSourceProperties dataSourceProperties = + new KafkaDataSourceProperties(originalDataSourceProperties); + Map alteredCustomProperties = Maps.newHashMap(); + alteredCustomProperties.put("client.id", "new-client"); + Deencapsulation.setField(dataSourceProperties, "customKafkaProperties", alteredCustomProperties); + dataSourceProperties.setKafkaPartitionOffsets(Lists.newArrayList(Pair.of(1, 20L))); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + try (MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { + kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets( + Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic-1"), Mockito.anyMap(), Mockito.anyList(), + Mockito.nullable(String.class))) + .thenReturn(Lists.newArrayList(Pair.of(1, 20L))); + Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); + } + + Assert.assertEquals("topic-1", routineLoadJob.getTopic()); + Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); + Map currentConvertedProperties = + Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); + Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); + Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); + Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + } + + @Test + public void testSuccessfulAlterUpdatesLoadDefinitionAndJournal() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + routineLoadJob.updateLoadDefinition(null); + Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, + LoadTask.MergeType.APPEND, "sequence_col"); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(routineLoadDesc); + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getEditLog()).thenReturn(editLog); + + routineLoadJob.modifyProperties(command); + + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + Assert.assertSame(routineLoadDesc, logCaptor.getValue().getRoutineLoadDesc()); + } + + Assert.assertEquals("sequence_col", routineLoadJob.getSequenceCol()); + Assert.assertNotSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + } + + @Test + public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(routineLoadJob, "desireTaskConcurrentNum", 1); + routineLoadJob.updateLoadDefinition(null); + + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( + new Separator(",", ","), null, null, null, null, null, null, + LoadTask.MergeType.APPEND, "sequence_col"); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); + routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( + routineLoadJob.getId(), jobProperties, null, routineLoadDesc)); + + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + connectContextStatic.close(); + connectContextStatic = null; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("db1"); + Mockito.when(database.getFullName()).thenReturn("db1"); + Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); + Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("table1"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + + RoutineLoadJob restored = imageRoundTrip(routineLoadJob); + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals("sequence_col", restored.getSequenceCol()); + Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals(2, (int) Deencapsulation.getField(restored, "desireTaskConcurrentNum")); + Assert.assertNotNull(Deencapsulation.getField(restored, "loadDefinition")); + } + } + + @Test + public void testImageRoundTripRestoresLegacyOrigStmt() throws Exception { + KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + String createSql = "CREATE ROUTINE LOAD db1.job1 ON stale_table " + + "COLUMNS TERMINATED BY ',' " + + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; + Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(createSql, 0)); + Deencapsulation.setField(routineLoadJob, "loadDefinition", null); + + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + connectContextStatic.close(); + connectContextStatic = null; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("db1"); + Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); + Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("table1"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + + RoutineLoadJob restored = imageRoundTrip(routineLoadJob); + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); + Assert.assertNull(Deencapsulation.getField(restored, "loadDefinition")); + } + } + + private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + routineLoadJob.write(out); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return RoutineLoadJob.read(in); + } + } + @Test public void testDisplayCustomPropertiesMasksKafkaSecrets() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 65aebd0084e729..a9face05891ab1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -19,18 +19,22 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.HashMap; import java.util.HashSet; @@ -229,6 +233,50 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi Assert.assertEquals("202", progress.getSequenceNumberByShard("shard-2")); } + @Test + public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { + KinesisRoutineLoadJob routineLoadJob = new KinesisRoutineLoadJob(1L, "job_atomic", 1L, + 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); + Map originalCustomProperties = Maps.newHashMap(); + originalCustomProperties.put("client.id", "old-client"); + Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); + Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); + Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); + Deencapsulation.setField(routineLoadJob, "customKinesisShards", Lists.newArrayList("shard-1")); + Map originalProgress = Maps.newHashMap(); + originalProgress.put("shard-1", "10"); + Deencapsulation.setField(routineLoadJob, "progress", new KinesisProgress(originalProgress)); + routineLoadJob.updateLoadDefinition(null); + Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + + Map originalDataSourceProperties = Maps.newHashMap(); + originalDataSourceProperties.put("property.client.id", "new-client"); + originalDataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), "us-east-1"); + KinesisDataSourceProperties dataSourceProperties = + new KinesisDataSourceProperties(originalDataSourceProperties); + Map alteredCustomProperties = Maps.newHashMap(); + alteredCustomProperties.put("client.id", "new-client"); + Deencapsulation.setField(dataSourceProperties, "customKinesisProperties", alteredCustomProperties); + Deencapsulation.setField(dataSourceProperties, "region", "us-east-1"); + dataSourceProperties.setKinesisShardPositions(Lists.newArrayList(Pair.of("shard-2", "20"))); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); + + Assert.assertEquals("ap-southeast-1", routineLoadJob.getRegion()); + Assert.assertEquals(Lists.newArrayList("shard-1"), + Deencapsulation.getField(routineLoadJob, "customKinesisShards")); + Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); + Map currentConvertedProperties = + Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); + Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); + Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); + Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + } + @Test public void testShardRefreshShouldMoveRetiredParentToClosedUntilConsumed() throws Exception { KinesisRoutineLoadJob routineLoadJob = diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 8a1550d48f5d13..522c8bee4d9948 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -18,7 +18,10 @@ package org.apache.doris.persist; import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; @@ -27,6 +30,8 @@ import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -60,8 +65,10 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, + LoadTask.MergeType.APPEND, "sequence_col"); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties); + jobProperties, routineLoadDataSourceProperties, routineLoadDesc); log.write(out); out.flush(); out.close(); @@ -81,9 +88,24 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); + Assert.assertEquals("sequence_col", log2.getRoutineLoadDesc().getSequenceColName()); in.close(); } + @Test + public void testDeserializeLegacyLogWithoutRoutineLoadDesc() throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + Text.writeString(out, "{\"jobId\":1000,\"jobProperties\":{}," + + "\"dataSourceProperties\":null}"); + } + + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); + Assert.assertEquals(1000L, log.getJobId()); + Assert.assertNull(log.getRoutineLoadDesc()); + } + } } From d0505b77ac1e64fdec04cc2d3b12c9cc54a3bf17 Mon Sep 17 00:00:00 2001 From: Refrain Date: Tue, 11 Aug 2026 11:00:45 +0800 Subject: [PATCH 02/11] [test](routineload) Define rollback compatibility boundary ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Keep the original CREATE statement in new images so an older FE can ignore the new load definition field and use its existing recovery path. ALTERed load clauses are outside the downgrade compatibility guarantee and may not survive rollback. ### Release note Document that Routine Load ALTER semantics are not guaranteed after rolling back to an older FE. ### Check List (For Author) - Test: Not run (per request; compatibility coverage was added) - Behavior changed: No. This records and tests the intended rollback compatibility boundary. - Does this need documentation: Yes. The rollback limitation must be called out in the feature documentation. --- .../load/routineload/RoutineLoadJob.java | 3 +- .../routineload/KafkaRoutineLoadJobTest.java | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 4229aca376353e..aefbb5e5d66013 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -263,7 +263,8 @@ public boolean isFinalState() { @SerializedName("ld") protected RoutineLoadDefinition loadDefinition; - // Legacy recovery input for images written before loadDefinition was persisted. + // Keep the original CREATE statement for downgrade compatibility. Older FEs ignore loadDefinition + // and restore from this field, so ALTER semantics are not guaranteed after rollback. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index c3cfad1dc1ff18..462803af886af1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -29,6 +29,7 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -56,6 +57,8 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import org.apache.kafka.common.PartitionInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -366,6 +369,10 @@ public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() thro 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); Deencapsulation.setField(routineLoadJob, "desireTaskConcurrentNum", 1); + String originalCreateSql = "CREATE ROUTINE LOAD db1.job1 ON table1 " + + "COLUMNS TERMINATED BY '|' " + + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; + Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(originalCreateSql, 0)); routineLoadJob.updateLoadDefinition(null); RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( @@ -406,6 +413,14 @@ public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() thro Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); Assert.assertEquals(2, (int) Deencapsulation.getField(restored, "desireTaskConcurrentNum")); Assert.assertNotNull(Deencapsulation.getField(restored, "loadDefinition")); + + // Simulate an older FE ignoring the unknown loadDefinition field. It can read the image + // through origStmt, but the ALTERed load clauses are intentionally not preserved on rollback. + RoutineLoadJob rollbackRestored = imageRoundTripWithoutLoadDefinition(routineLoadJob); + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, rollbackRestored.getState()); + Assert.assertEquals("|", rollbackRestored.getColumnSeparator().getSeparator()); + Assert.assertNull(rollbackRestored.getSequenceCol()); + Assert.assertNull(Deencapsulation.getField(rollbackRestored, "loadDefinition")); } } @@ -460,6 +475,29 @@ private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) thro } } + private static RoutineLoadJob imageRoundTripWithoutLoadDefinition(RoutineLoadJob routineLoadJob) + throws Exception { + ByteArrayOutputStream image = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(image)) { + routineLoadJob.write(out); + } + + String json; + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image.toByteArray()))) { + json = Text.readString(in); + } + JsonObject jobJson = JsonParser.parseString(json).getAsJsonObject(); + Assert.assertNotNull(jobJson.remove("ld")); + + ByteArrayOutputStream legacyImage = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(legacyImage)) { + Text.writeString(out, jobJson.toString()); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(legacyImage.toByteArray()))) { + return RoutineLoadJob.read(in); + } + } + @Test public void testDisplayCustomPropertiesMasksKafkaSecrets() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, From aac38e8db61826ac15afc6f0912055edd11c09cf Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 14 Aug 2026 14:17:49 +0800 Subject: [PATCH 03/11] temp --- .../routineload/RoutineLoadDefinition.java | 128 ------ .../load/routineload/RoutineLoadJob.java | 313 +++++++------ .../kafka/KafkaRoutineLoadJob.java | 196 +++----- .../kinesis/KinesisRoutineLoadJob.java | 197 +++----- .../routineload/KafkaRoutineLoadJobTest.java | 229 ++-------- .../KinesisRoutineLoadJobTest.java | 131 ++++-- .../RoutineLoadJobPersistenceTest.java | 426 ++++++++++++++++++ .../AlterRoutineLoadOperationLogTest.java | 100 ++-- 8 files changed, 922 insertions(+), 798 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java deleted file mode 100644 index bcfb59c48d90f4..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadDefinition.java +++ /dev/null @@ -1,128 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.load.routineload; - -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; -import org.apache.doris.analysis.ToSqlParams; -import org.apache.doris.common.UserException; -import org.apache.doris.load.RoutineLoadDesc; -import org.apache.doris.load.loadv2.LoadTask; -import org.apache.doris.nereids.load.NereidsLoadUtils; -import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo; -import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadColumnDesc; -import org.apache.doris.nereids.trees.plans.commands.load.LoadDeleteOnClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadPartitionNames; -import org.apache.doris.nereids.trees.plans.commands.load.LoadPrecedingFilterClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; -import org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator; -import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause; -import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause; -import org.apache.doris.nereids.trees.expressions.Expression; - -import com.google.common.collect.Maps; -import com.google.gson.annotations.SerializedName; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Snapshot of the current CREATE ROUTINE LOAD semantics. - */ -public class RoutineLoadDefinition { - @SerializedName("desc") - private RoutineLoadDesc routineLoadDesc; - @SerializedName("jp") - private Map jobProperties = Maps.newHashMap(); - @SerializedName("dsp") - private Map dataSourceProperties = Maps.newHashMap(); - - public RoutineLoadDefinition(RoutineLoadDesc routineLoadDesc, - Map jobProperties, Map dataSourceProperties) { - this.routineLoadDesc = routineLoadDesc; - this.jobProperties.putAll(jobProperties); - this.dataSourceProperties.putAll(dataSourceProperties); - } - - public RoutineLoadDesc getRoutineLoadDesc() { - return routineLoadDesc; - } - - public Map getDataSourceProperties() { - return dataSourceProperties; - } - - public CreateRoutineLoadInfo toCreateInfo(String dbName, String jobName, String tableName, - LoadDataSourceType dataSourceType, String comment) throws UserException { - LoadTask.MergeType mergeType = routineLoadDesc == null - ? LoadTask.MergeType.APPEND : routineLoadDesc.getMergeType(); - return new CreateRoutineLoadInfo(new LabelNameInfo(dbName, jobName), tableName, - toLoadPropertyMap(routineLoadDesc), Maps.newHashMap(jobProperties), dataSourceType.name(), - Maps.newHashMap(dataSourceProperties), mergeType, comment); - } - - private static Map toLoadPropertyMap(RoutineLoadDesc routineLoadDesc) throws UserException { - Map loadProperties = Maps.newHashMap(); - if (routineLoadDesc == null) { - return loadProperties; - } - if (routineLoadDesc.getColumnSeparator() != null) { - put(loadProperties, new LoadSeparator(routineLoadDesc.getColumnSeparator().getOriSeparator())); - } - if (routineLoadDesc.getColumnsInfo() != null) { - List columns = new ArrayList<>(); - for (ImportColumnDesc column : routineLoadDesc.getColumnsInfo()) { - Expression expression = column.getExpr() == null ? null : parseExpression(column.getExpr()); - columns.add(new LoadColumnDesc(column.getColumnName(), expression)); - } - put(loadProperties, new LoadColumnClause(columns)); - } - if (routineLoadDesc.getPrecedingFilter() != null) { - put(loadProperties, new LoadPrecedingFilterClause( - parseExpression(routineLoadDesc.getPrecedingFilter()))); - } - if (routineLoadDesc.getFilter() != null) { - put(loadProperties, new LoadWhereClause(parseExpression(routineLoadDesc.getFilter()))); - } - if (routineLoadDesc.getPartitionNamesInfo() != null) { - put(loadProperties, new LoadPartitionNames( - routineLoadDesc.getPartitionNamesInfo().isTemp(), - routineLoadDesc.getPartitionNamesInfo().getPartitionNames())); - } - if (routineLoadDesc.getDeleteCondition() != null) { - put(loadProperties, new LoadDeleteOnClause(parseExpression(routineLoadDesc.getDeleteCondition()))); - } - if (routineLoadDesc.hasSequenceCol()) { - put(loadProperties, new LoadSequenceClause(routineLoadDesc.getSequenceColName())); - } - return loadProperties; - } - - private static Expression parseExpression(Expr expression) throws UserException { - String sql = expression.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - return NereidsLoadUtils.parseExpressionSeq(sql).get(0); - } - - private static void put(Map loadProperties, LoadProperty loadProperty) { - loadProperties.put(loadProperty.getClass().getName(), loadProperty); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index aefbb5e5d66013..57497876059a81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -117,6 +116,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); + private static final int CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION = 1; public static final long DEFAULT_MAX_ERROR_NUM = 0; public static final double DEFAULT_MAX_FILTER_RATIO = 1.0; @@ -180,14 +180,23 @@ public boolean isFinalState() { protected long dbId; @SerializedName("tbid") protected long tableId; + // An absent version identifies a legacy record whose CREATE statement still needs to be migrated. + @SerializedName("rlpv") + private int routineLoadPersistenceVersion; // this code is used to verify be task request protected long authCode; // protected RoutineLoadDesc routineLoadDesc; // optional + @SerializedName("pni") protected PartitionNamesInfo partitionNamesInfo; // optional + @SerializedName("cds") protected ImportColumnDescs columnDescs; // optional + @SerializedName("pf") protected Expr precedingFilter; // optional + @SerializedName("we") protected Expr whereExpr; // optional + @SerializedName("cs") protected Separator columnSeparator; // optional + @SerializedName("lidel") protected Separator lineDelimiter; @SerializedName("dtcn") protected int desireTaskConcurrentNum; // optional @@ -202,6 +211,7 @@ public boolean isFinalState() { @SerializedName("men") protected long maxErrorNum = DEFAULT_MAX_ERROR_NUM; // optional protected double maxFilterRatio = DEFAULT_MAX_FILTER_RATIO; + @SerializedName("eml") protected long execMemLimit = DEFAULT_EXEC_MEM_LIMIT; protected int sendBatchParallelism = DEFAULT_SEND_BATCH_PARALLELISM; protected boolean loadToSingleTablet = DEFAULT_LOAD_TO_SINGLE_TABLET; @@ -231,8 +241,10 @@ public boolean isFinalState() { protected TPartialUpdateNewRowPolicy partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; protected TUniqueKeyUpdateMode uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; + @SerializedName("sc") protected String sequenceCol; + @SerializedName("mosn") protected boolean memtableOnSinkNode = false; protected int currentTaskConcurrentNum; @@ -259,12 +271,7 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // Canonical current CREATE semantics. CREATE and ALTER must keep this snapshot current. - @SerializedName("ld") - protected RoutineLoadDefinition loadDefinition; - - // Keep the original CREATE statement for downgrade compatibility. Older FEs ignore loadDefinition - // and restore from this field, so ALTER semantics are not guaranteed after rollback. + // Keep the original CREATE statement for downgrade compatibility and legacy image migration. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -275,7 +282,9 @@ public boolean isFinalState() { protected String comment = ""; protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + @SerializedName("mt") protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete + @SerializedName("dc") protected Expr deleteCondition; // TODO(ml): error sample @@ -321,6 +330,7 @@ public RoutineLoadJob(Long id, String name, this.tableId = tableId; this.authCode = 0; this.userIdentity = userIdentity; + this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -350,6 +360,7 @@ public RoutineLoadJob(Long id, String name, this.authCode = 0; this.userIdentity = userIdentity; this.isMultiTable = true; + this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -475,62 +486,6 @@ protected void setRoutineLoadDesc(RoutineLoadDesc routineLoadDesc) { } } - protected RoutineLoadDesc getLoadDefinitionRoutineLoadDesc() { - List columnsInfo = columnDescs == null ? null : columnDescs.descs; - return new RoutineLoadDesc(columnSeparator, lineDelimiter, columnsInfo, precedingFilter, whereExpr, - partitionNamesInfo, deleteCondition, mergeType, sequenceCol); - } - - protected void initializeLoadDefinition(CreateRoutineLoadInfo info) { - Map originalDataSourceProperties = - info.getDataSourceProperties().getOriginalDataSourceProperties(); - Map dataSourceProperties = originalDataSourceProperties == null - ? Maps.newHashMap() : Maps.newHashMap(originalDataSourceProperties); - updateLoadDefinitionDataSourceProperties(dataSourceProperties); - loadDefinition = new RoutineLoadDefinition( - getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); - } - - protected void updateLoadDefinition(AbstractDataSourceProperties changedDataSourceProperties) { - Map dataSourceProperties = loadDefinition == null - ? Maps.newHashMap() : Maps.newHashMap(loadDefinition.getDataSourceProperties()); - if (changedDataSourceProperties != null - && changedDataSourceProperties.getOriginalDataSourceProperties() != null) { - dataSourceProperties.putAll(changedDataSourceProperties.getOriginalDataSourceProperties()); - } - updateLoadDefinitionDataSourceProperties(dataSourceProperties); - loadDefinition = new RoutineLoadDefinition( - getLoadDefinitionRoutineLoadDesc(), snapshotLoadDefinitionJobProperties(), dataSourceProperties); - } - - protected Map snapshotLoadDefinitionJobProperties() { - Map currentJobProperties = Maps.newHashMap(jobProperties); - currentJobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, - String.valueOf(desireTaskConcurrentNum)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_ERROR_NUMBER_PROPERTY, String.valueOf(maxErrorNum)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, String.valueOf(maxFilterRatio)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_INTERVAL_SEC_PROPERTY, - String.valueOf(maxBatchIntervalS)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_ROWS_PROPERTY, String.valueOf(maxBatchRows)); - currentJobProperties.put(CreateRoutineLoadInfo.MAX_BATCH_SIZE_PROPERTY, String.valueOf(maxBatchSizeBytes)); - currentJobProperties.put(CreateRoutineLoadInfo.EXEC_MEM_LIMIT_PROPERTY, String.valueOf(execMemLimit)); - currentJobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, - String.valueOf(sendBatchParallelism)); - currentJobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, - String.valueOf(loadToSingleTablet)); - currentJobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); - currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); - if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - currentJobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - } else { - currentJobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, - partialUpdateNewKeyPolicy.name()); - } - return currentJobProperties; - } - - protected abstract void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties); - @Override public long getId() { return id; @@ -2028,85 +1983,105 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } - // Process UNIQUE_KEY_UPDATE_MODE first to ensure correct backward compatibility - // with PARTIAL_COLUMNS (HashMap iteration order is not guaranteed) - if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - String modeValue = jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode(modeValue); - if (mode != null) { - uniqueKeyUpdateMode = mode; - isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); - } else { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; - } + if (routineLoadPersistenceVersion == 0) { + // Legacy images did not persist this create-time session option. Preserve their historical + // post-restart behavior instead of inheriting the image-loading thread's ConnectContext. + memtableOnSinkNode = false; } - // Process remaining properties - jobProperties.forEach((k, v) -> { - if (k.equals(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - // Backward compatibility: only use partial_columns if unique_key_update_mode is not set - // unique_key_update_mode takes precedence - if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - isPartialUpdate = Boolean.parseBoolean(v); - if (isPartialUpdate) { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - } - } else if (k.equals(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - if ("ERROR".equalsIgnoreCase(v)) { - partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - }); try { - ConnectContext ctx = new ConnectContext(); - ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); - StatementContext statementContext = new StatementContext(); - statementContext.setConnectContext(ctx); - ctx.setStatementContext(statementContext); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setCurrentUserIdentity(UserIdentity.ADMIN); - ctx.getState().reset(); - try { - ctx.setThreadLocalInfo(); - if (loadDefinition == null) { - restoreLegacyDefinition(ctx); - } else { - restoreLoadDefinition(ctx); - } - } finally { - ctx.cleanup(); + hydrateJobProperties(); + if (routineLoadPersistenceVersion == 0) { + restoreLegacyDefinition(); + routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; } } catch (Exception e) { this.state = JobState.CANCELLED; - LOG.warn("error happens when restoring routine load definition", e); + LOG.warn("error happens when restoring routine load job", e); } if (userIdentity != null) { userIdentity.setIsAnalyzed(); } } - private void restoreLegacyDefinition(ConnectContext ctx) throws UserException { - NereidsParser nereidsParser = new NereidsParser(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( - origStmt.originStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - if (!isMultiTable) { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); - createRoutineLoadInfo.setTableName(db.getTable(tableId).get().getName()); + private void hydrateJobProperties() throws UserException { + if (jobProperties.containsKey(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)) { + maxFilterRatio = Double.parseDouble( + jobProperties.get(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)) { + sendBatchParallelism = Integer.parseInt( + jobProperties.get(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)) { + loadToSingleTablet = Boolean.parseBoolean( + jobProperties.get(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)); + } + + boolean hasUniqueKeyUpdateMode = jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + if (hasUniqueKeyUpdateMode) { + TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + uniqueKeyUpdateMode = mode == null ? TUniqueKeyUpdateMode.UPSERT : mode; + isPartialUpdate = uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + if (!hasUniqueKeyUpdateMode && jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + isPartialUpdate = Boolean.parseBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + if (isPartialUpdate) { + uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase( + jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) + ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; + } + + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { + enclose = parseEnclose(jobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { + escape = parseEscape(jobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + emptyFieldAsNull = Boolean.parseBoolean( + jobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); } - private void restoreLoadDefinition(ConnectContext ctx) throws UserException { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); - String tableName = isMultiTable ? null : db.getTable(tableId).get().getName(); - CreateRoutineLoadInfo createRoutineLoadInfo = loadDefinition.toCreateInfo( - db.getFullName(), name, tableName, dataSourceType, comment); - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(loadDefinition.getRoutineLoadDesc()); + private void restoreLegacyDefinition() throws UserException { + ConnectContext ctx = new ConnectContext(); + ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); + StatementContext statementContext = new StatementContext(); + statementContext.setConnectContext(ctx); + ctx.setStatementContext(statementContext); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.getState().reset(); + try { + ctx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( + origStmt.originStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + // Resolve the current table name by ID so table rename or SWAP TABLE does not leave the + // legacy CREATE statement pointing at a stale table name. + if (!isMultiTable && tableId != 0) { + try { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); + if (db != null) { + db.getTable(tableId).ifPresent( + table -> createRoutineLoadInfo.setTableName(table.getName())); + } + } catch (Exception ignored) { + // Let validate() below surface the original catalog error. + } + } + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); + execMemLimit = createRoutineLoadInfo.getExecMemLimit(); + } finally { + ctx.cleanup(); + } } public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; @@ -2115,22 +2090,27 @@ private void restoreLoadDefinition(ConnectContext ctx) throws UserException { public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; - protected TUniqueKeyUpdateMode validateCommonJobProperties(Map jobProperties) - throws UserException { - if (!jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - return null; + protected void validateCommonJobProperties(Map jobProperties) throws UserException { + validateCsvFormatProperties(jobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } } - TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( - jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + if (!"APPEND".equalsIgnoreCase(policy) && !"ERROR".equalsIgnoreCase(policy)) { + throw new AnalysisException(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY + + " should be one of {'APPEND', 'ERROR'}, but found " + policy); + } } - return newMode; } - // for ALTER ROUTINE LOAD. All failure-prone validation must be completed before calling this method. - protected void modifyCommonJobProperties(Map jobProperties, - TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode) { + // for ALTER ROUTINE LOAD. Validate all common properties before changing any common runtime state. + protected void modifyCommonJobProperties(Map jobProperties) throws UserException { + validateCommonJobProperties(jobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); @@ -2163,8 +2143,8 @@ protected void modifyCommonJobProperties(Map jobProperties, } if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - this.uniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; + String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + this.uniqueKeyUpdateMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); @@ -2181,6 +2161,55 @@ protected void modifyCommonJobProperties(Map jobProperties, this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); } + + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase(policy) + ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; + this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, + partialUpdateNewKeyPolicy.name()); + } + + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_ENCLOSE); + enclose = parseEnclose(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, value); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_ESCAPE); + escape = parseEscape(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, value); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL); + emptyFieldAsNull = Boolean.parseBoolean(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, value); + } + } + + private static void validateCsvFormatProperties(Map jobProperties) { + if (!jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE) + && !jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE) + && !jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + return; + } + Map csvProperties = Maps.newHashMap(); + for (String property : new String[] {CsvFileFormatProperties.PROP_ENCLOSE, + CsvFileFormatProperties.PROP_ESCAPE, CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL}) { + if (jobProperties.containsKey(property)) { + csvProperties.put(property, jobProperties.get(property)); + } + } + CsvFileFormatProperties properties = new CsvFileFormatProperties(FileFormatProperties.FORMAT_CSV); + properties.analyzeFileFormatProperties(csvProperties, false); + } + + private static byte parseEnclose(String value) { + return Strings.isNullOrEmpty(value) ? 0 : (byte) value.charAt(0); + } + + private static byte parseEscape(String value) { + return Strings.isNullOrEmpty(value) ? 0 : value.getBytes()[0]; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index a5d1dd5f0e7ef7..be124f7c72e6c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -60,8 +60,6 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -75,7 +73,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -217,25 +214,19 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - Pair, String> convertedProperties = buildConvertedCustomProperties( - customProperties, kafkaDefaultOffSet); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(convertedProperties.first); - kafkaDefaultOffSet = convertedProperties.second; - } + if (rebuild) { + convertedCustomProperties.clear(); + } - private Pair, String> buildConvertedCustomProperties( - Map sourceProperties, String currentDefaultOffset) throws DdlException { - Map convertedProperties = Maps.newHashMap(); - for (Map.Entry entry : sourceProperties.entrySet()) { + SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); + for (Map.Entry entry : customProperties.entrySet()) { if (entry.getValue().startsWith("FILE:")) { // convert FILE:file_name -> FILE:file_id:md5 String file = entry.getValue().substring(entry.getValue().indexOf(":") + 1); - SmallFileMgr smallFileMgr = Env.getCurrentEnv().getSmallFileMgr(); SmallFile smallFile = smallFileMgr.getSmallFile(dbId, KAFKA_FILE_CATALOG, file, true); - convertedProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); + convertedCustomProperties.put(entry.getKey(), "FILE:" + smallFile.id + ":" + smallFile.md5); } else { - convertedProperties.put(entry.getKey(), entry.getValue()); + convertedCustomProperties.put(entry.getKey(), entry.getValue()); } } @@ -244,14 +235,14 @@ private Pair, String> buildConvertedCustomProperties( // KAFKA_DEFAULT_OFFSETS, and this attribute will be converted into a timestamp during the analyzing phase, // thus losing some information. So we use KAFKA_ORIGIN_DEFAULT_OFFSETS to store the original datetime // formatted KAFKA_DEFAULT_OFFSETS value - String convertedDefaultOffset = currentDefaultOffset; - if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { - convertedDefaultOffset = convertedProperties + if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())) { + kafkaDefaultOffSet = convertedCustomProperties .remove(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName()); - } else if (convertedProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { - convertedDefaultOffset = convertedProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); + return; + } + if (convertedCustomProperties.containsKey(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName())) { + kafkaDefaultOffSet = convertedCustomProperties.remove(KafkaConfiguration.KAFKA_DEFAULT_OFFSETS.getName()); } - return Pair.of(convertedProperties, convertedDefaultOffset); } @Override @@ -597,7 +588,6 @@ public static KafkaRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, Con kafkaRoutineLoadJob.setOptional(info); kafkaRoutineLoadJob.checkCustomProperties(); kafkaRoutineLoadJob.checkCustomPartition(); - kafkaRoutineLoadJob.initializeLoadDefinition(info); return kafkaRoutineLoadJob; } @@ -783,24 +773,10 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } - @Override - protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { - dataSourceProperties.put(KafkaConfiguration.KAFKA_BROKER_LIST.getName(), brokerList); - dataSourceProperties.put(KafkaConfiguration.KAFKA_TOPIC.getName(), topic); - dataSourceProperties.remove(KafkaConfiguration.KAFKA_OFFSETS.getName()); - if (customKafkaPartitions.isEmpty()) { - dataSourceProperties.remove(KafkaConfiguration.KAFKA_PARTITIONS.getName()); - } else { - dataSourceProperties.put(KafkaConfiguration.KAFKA_PARTITIONS.getName(), - Joiner.on(",").join(customKafkaPartitions)); - } - customProperties.forEach((key, value) -> dataSourceProperties.put( - key.startsWith("aws.") ? key : "property." + key, value)); - } - @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); + validateCommonJobProperties(jobProperties); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); if (null != dataSourceProperties) { // if the partition offset is set by timestamp, convert it to real offset @@ -815,7 +791,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); - updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); @@ -845,122 +820,70 @@ private void convertOffset(KafkaDataSourceProperties dataSourceProperties) throw private void modifyPropertiesInternal(Map jobProperties, KafkaDataSourceProperties dataSourceProperties) throws UserException { - PreparedKafkaAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); - applyAlter(jobProperties, dataSourceProperties, preparedAlter); - if (LOG.isDebugEnabled()) { - LOG.debug("modify the properties of kafka routine load job: {}, jobProperties: {}, " - + "datasource properties: {}", - this.id, jobProperties, dataSourceProperties); - } - } + if (null != dataSourceProperties) { + List> kafkaPartitionOffsets = Lists.newArrayList(); + Map customKafkaProperties = Maps.newHashMap(); - private PreparedKafkaAlter prepareAlter(Map jobProperties, - KafkaDataSourceProperties dataSourceProperties) throws UserException { - TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); - List> kafkaPartitionOffsets = Lists.newArrayList(); - Map alteredCustomProperties = Maps.newHashMap(); - if (dataSourceProperties != null - && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); - alteredCustomProperties = dataSourceProperties.getCustomKafkaProperties(); - } + if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + kafkaPartitionOffsets = dataSourceProperties.getKafkaPartitionOffsets(); + customKafkaProperties = dataSourceProperties.getCustomKafkaProperties(); + } - Map stagedCustomProperties = null; - Map stagedConvertedCustomProperties = null; - String stagedKafkaDefaultOffset = kafkaDefaultOffSet; - if (!alteredCustomProperties.isEmpty()) { - stagedCustomProperties = Maps.newHashMap(customProperties); - stagedCustomProperties.putAll(alteredCustomProperties); - Pair, String> convertedProperties = buildConvertedCustomProperties( - stagedCustomProperties, stagedKafkaDefaultOffset); - stagedConvertedCustomProperties = convertedProperties.first; - stagedKafkaDefaultOffset = convertedProperties.second; - } + // convertCustomProperties and check partitions before reset progress to make modify operation atomic + if (!customKafkaProperties.isEmpty()) { + this.customProperties.putAll(customKafkaProperties); + convertCustomProperties(true); + } - if (!kafkaPartitionOffsets.isEmpty()) { - ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); - } - if (dataSourceProperties != null && Config.isCloudMode()) { - Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() - .setRequestIp(FrontendOptions.getLocalHostAddressCached()); - builder.setCloudUniqueId(Config.cloud_unique_id); - builder.setDbId(dbId); - builder.setJobId(id); if (!kafkaPartitionOffsets.isEmpty()) { - Map partitionOffsetMap = new HashMap<>(); - for (Pair pair : kafkaPartitionOffsets) { - // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is - // this value will be incremented when pulling MS persistent progress data and updating memory - // in routineLoadJob.updateCloudProgress(). - partitionOffsetMap.put(pair.first, pair.second - 1); - } - builder.putAllPartitionToOffset(partitionOffsetMap); + ((KafkaProgress) progress).checkPartitions(kafkaPartitionOffsets); } - resetCloudProgress(builder); - } - return new PreparedKafkaAlter(validatedUniqueKeyUpdateMode, kafkaPartitionOffsets, - stagedCustomProperties, stagedConvertedCustomProperties, stagedKafkaDefaultOffset); - } - private void applyAlter(Map jobProperties, KafkaDataSourceProperties dataSourceProperties, - PreparedKafkaAlter preparedAlter) { - if (dataSourceProperties != null) { - if (preparedAlter.stagedCustomProperties != null) { - customProperties.clear(); - customProperties.putAll(preparedAlter.stagedCustomProperties); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); - kafkaDefaultOffSet = preparedAlter.stagedKafkaDefaultOffset; + if (Config.isCloudMode()) { + Cloud.ResetRLProgressRequest.Builder builder = Cloud.ResetRLProgressRequest.newBuilder() + .setRequestIp(FrontendOptions.getLocalHostAddressCached()); + builder.setCloudUniqueId(Config.cloud_unique_id); + builder.setDbId(dbId); + builder.setJobId(id); + if (!kafkaPartitionOffsets.isEmpty()) { + Map partitionOffsetMap = new HashMap<>(); + for (Pair pair : kafkaPartitionOffsets) { + // The reason why the value recorded in MS in cloud mode needs to be subtracted by one is + // this value will be incremented + // when pulling MS persistent progress data and updating memory + // in routineLoadJob.updateCloudProgress(). + partitionOffsetMap.put(pair.first, pair.second - 1); + } + builder.putAllPartitionToOffset(partitionOffsetMap); + } + resetCloudProgress(builder); } + // It is necessary to reset the Kafka progress cache if topic change, // and should reset cache before modifying partition offset. if (!Strings.isNullOrEmpty(dataSourceProperties.getTopic())) { - topic = dataSourceProperties.getTopic(); - progress = new KafkaProgress(); + this.topic = dataSourceProperties.getTopic(); + this.progress = new KafkaProgress(); } - if (!preparedAlter.kafkaPartitionOffsets.isEmpty()) { - ((KafkaProgress) progress).modifyOffset(preparedAlter.kafkaPartitionOffsets); + + // modify partition offset + if (!kafkaPartitionOffsets.isEmpty()) { + // we can only modify the partition that is being consumed + ((KafkaProgress) progress).modifyOffset(kafkaPartitionOffsets); } + + // modify broker list if (!Strings.isNullOrEmpty(dataSourceProperties.getBrokerList())) { - brokerList = dataSourceProperties.getBrokerList(); + this.brokerList = dataSourceProperties.getBrokerList(); } } if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); + modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if ("ERROR".equalsIgnoreCase(policy)) { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - } - } - - private static class PreparedKafkaAlter { - private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; - private final List> kafkaPartitionOffsets; - private final Map stagedCustomProperties; - private final Map stagedConvertedCustomProperties; - private final String stagedKafkaDefaultOffset; - - private PreparedKafkaAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, - List> kafkaPartitionOffsets, - Map stagedCustomProperties, - Map stagedConvertedCustomProperties, - String stagedKafkaDefaultOffset) { - this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; - this.kafkaPartitionOffsets = kafkaPartitionOffsets; - this.stagedCustomProperties = stagedCustomProperties; - this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; - this.stagedKafkaDefaultOffset = stagedKafkaDefaultOffset; } + LOG.info("modify the properties of kafka routine load job: {}, jobProperties: {}, datasource properties: {}", + this.id, jobProperties, dataSourceProperties); } private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) throws DdlException { @@ -987,7 +910,6 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); setRoutineLoadDesc(log.getRoutineLoadDesc()); - updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 5ca8355d851c50..9c5bac1c7895a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -51,8 +51,6 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -66,7 +64,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -187,20 +184,19 @@ private void convertCustomProperties(boolean rebuild) throws DdlException { return; } - Pair, String> convertedProperties = buildConvertedCustomProperties( - customProperties, kinesisDefaultPosition); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(convertedProperties.first); - kinesisDefaultPosition = convertedProperties.second; - } + if (rebuild) { + convertedCustomProperties.clear(); + } + + for (Map.Entry entry : customProperties.entrySet()) { + convertedCustomProperties.put(entry.getKey(), entry.getValue()); + } - private Pair, String> buildConvertedCustomProperties( - Map sourceProperties, String currentDefaultPosition) { - Map convertedProperties = Maps.newHashMap(sourceProperties); - String convertedDefaultPosition = convertedProperties.getOrDefault( - "kinesis_default_pos", currentDefaultPosition); - // Keep kinesis_default_pos in convertedProperties so BE can use it. - return Pair.of(convertedProperties, convertedDefaultPosition); + // Handle default position + if (convertedCustomProperties.containsKey("kinesis_default_pos")) { + kinesisDefaultPosition = convertedCustomProperties.get("kinesis_default_pos"); + // Keep it in convertedCustomProperties so BE can use it + } } private String convertedDefaultPosition() { @@ -534,7 +530,6 @@ public static KinesisRoutineLoadJob fromCreateInfo(CreateRoutineLoadInfo info, C kinesisRoutineLoadJob.setOptional(info); kinesisRoutineLoadJob.checkCustomProperties(); - kinesisRoutineLoadJob.initializeLoadDefinition(info); return kinesisRoutineLoadJob; } @@ -663,27 +658,6 @@ public Map getCustomProperties() { return getMaskedCustomProperties("property."); } - @Override - protected void updateLoadDefinitionDataSourceProperties(Map dataSourceProperties) { - dataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), region); - dataSourceProperties.put(KinesisConfiguration.KINESIS_STREAM.getName(), stream); - if (endpoint == null) { - dataSourceProperties.remove(KinesisConfiguration.KINESIS_ENDPOINT.getName()); - dataSourceProperties.remove("kinesis_endpoint"); - } else { - dataSourceProperties.put(KinesisConfiguration.KINESIS_ENDPOINT.getName(), endpoint); - } - dataSourceProperties.remove(KinesisConfiguration.KINESIS_POSITIONS.getName()); - if (customKinesisShards.isEmpty()) { - dataSourceProperties.remove(KinesisConfiguration.KINESIS_SHARDS.getName()); - } else { - dataSourceProperties.put(KinesisConfiguration.KINESIS_SHARDS.getName(), - Joiner.on(",").join(customKinesisShards)); - } - customProperties.forEach((key, value) -> dataSourceProperties.put( - key.startsWith("aws.") ? key : "property." + key, value)); - } - private Map getMaskedCustomProperties(String keyPrefix) { Map maskedProperties = new HashMap<>(); customProperties.forEach((key, value) -> { @@ -712,7 +686,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); - updateLoadDefinition(dataSourceProperties); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); @@ -725,120 +698,73 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti private void modifyPropertiesInternal(Map jobProperties, KinesisDataSourceProperties dataSourceProperties) throws UserException { - PreparedKinesisAlter preparedAlter = prepareAlter(jobProperties, dataSourceProperties); - applyAlter(jobProperties, dataSourceProperties, preparedAlter); - LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", - this.id, jobProperties, dataSourceProperties); - } - - private PreparedKinesisAlter prepareAlter(Map jobProperties, - KinesisDataSourceProperties dataSourceProperties) throws UserException { - TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode = validateCommonJobProperties(jobProperties); - List> shardPositions = Lists.newArrayList(); - Map alteredCustomProperties = Maps.newHashMap(); - if (dataSourceProperties != null - && MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { - shardPositions = dataSourceProperties.getKinesisShardPositions(); - alteredCustomProperties = dataSourceProperties.getCustomKinesisProperties(); - } - - Map stagedCustomProperties = null; - Map stagedConvertedCustomProperties = null; - String stagedDefaultPosition = kinesisDefaultPosition; - if (!alteredCustomProperties.isEmpty()) { - stagedCustomProperties = Maps.newHashMap(customProperties); - stagedCustomProperties.putAll(alteredCustomProperties); - Pair, String> convertedProperties = buildConvertedCustomProperties( - stagedCustomProperties, stagedDefaultPosition); - stagedConvertedCustomProperties = convertedProperties.first; - stagedDefaultPosition = convertedProperties.second; - } - - boolean resetProgress = dataSourceProperties != null - && !Strings.isNullOrEmpty(dataSourceProperties.getStream()); - if (!shardPositions.isEmpty() && !resetProgress) { - ((KinesisProgress) progress).checkShards(shardPositions); - } - return new PreparedKinesisAlter(validatedUniqueKeyUpdateMode, shardPositions, - stagedCustomProperties, stagedConvertedCustomProperties, stagedDefaultPosition, resetProgress); - } - - private void applyAlter(Map jobProperties, KinesisDataSourceProperties dataSourceProperties, - PreparedKinesisAlter preparedAlter) { + validateCommonJobProperties(jobProperties); if (dataSourceProperties != null) { - if (preparedAlter.stagedCustomProperties != null) { - customProperties.clear(); - customProperties.putAll(preparedAlter.stagedCustomProperties); - convertedCustomProperties.clear(); - convertedCustomProperties.putAll(preparedAlter.stagedConvertedCustomProperties); - kinesisDefaultPosition = preparedAlter.stagedDefaultPosition; + List> shardPositions = Lists.newArrayList(); + Map customKinesisProperties = Maps.newHashMap(); + boolean resetProgress = false; + boolean hasExplicitShardPositions = false; + + if (MapUtils.isNotEmpty(dataSourceProperties.getOriginalDataSourceProperties())) { + shardPositions = dataSourceProperties.getKinesisShardPositions(); + customKinesisProperties = dataSourceProperties.getCustomKinesisProperties(); + hasExplicitShardPositions = !shardPositions.isEmpty(); + } + + // Update custom properties + if (!customKinesisProperties.isEmpty()) { + this.customProperties.putAll(customKinesisProperties); + convertCustomProperties(true); } + + // Modify stream if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getStream())) { - stream = dataSourceProperties.getStream(); + this.stream = dataSourceProperties.getStream(); + resetProgress = true; } + + // Modify region if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getRegion())) { - region = dataSourceProperties.getRegion(); + this.region = dataSourceProperties.getRegion(); } + + // Modify endpoint if provided if (!Strings.isNullOrEmpty(dataSourceProperties.getEndpoint())) { - endpoint = dataSourceProperties.getEndpoint(); + this.endpoint = dataSourceProperties.getEndpoint(); } - if (preparedAlter.resetProgress) { - progress = new KinesisProgress(); - openKinesisShards.clear(); - closedKinesisShards.clear(); - cachedShardWithMillsBehindLatest.clear(); + + if (resetProgress) { + this.progress = new KinesisProgress(); + this.openKinesisShards.clear(); + this.closedKinesisShards.clear(); + this.cachedShardWithMillsBehindLatest.clear(); } - if (!preparedAlter.shardPositions.isEmpty()) { - customKinesisShards.clear(); - for (Pair shardPosition : preparedAlter.shardPositions) { - customKinesisShards.add(shardPosition.first); + + if (hasExplicitShardPositions) { + this.customKinesisShards.clear(); + for (Pair shardPosition : shardPositions) { + this.customKinesisShards.add(shardPosition.first); } - } else if (preparedAlter.resetProgress) { + } else if (resetProgress) { // Stream change without explicit shards should fall back to dynamic shard discovery. - customKinesisShards.clear(); + this.customKinesisShards.clear(); } - if (!preparedAlter.shardPositions.isEmpty()) { - ((KinesisProgress) progress).modifyPosition(preparedAlter.shardPositions); + + if (!shardPositions.isEmpty()) { + if (!resetProgress) { + ((KinesisProgress) progress).checkShards(shardPositions); + } + ((KinesisProgress) progress).modifyPosition(shardPositions); } } + if (!jobProperties.isEmpty()) { Map copiedJobProperties = Maps.newHashMap(jobProperties); - modifyCommonJobProperties(copiedJobProperties, preparedAlter.validatedUniqueKeyUpdateMode); + modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if ("ERROR".equalsIgnoreCase(policy)) { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - } - } - - private static class PreparedKinesisAlter { - private final TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode; - private final List> shardPositions; - private final Map stagedCustomProperties; - private final Map stagedConvertedCustomProperties; - private final String stagedDefaultPosition; - private final boolean resetProgress; - - private PreparedKinesisAlter(TUniqueKeyUpdateMode validatedUniqueKeyUpdateMode, - List> shardPositions, - Map stagedCustomProperties, - Map stagedConvertedCustomProperties, - String stagedDefaultPosition, boolean resetProgress) { - this.validatedUniqueKeyUpdateMode = validatedUniqueKeyUpdateMode; - this.shardPositions = shardPositions; - this.stagedCustomProperties = stagedCustomProperties; - this.stagedConvertedCustomProperties = stagedConvertedCustomProperties; - this.stagedDefaultPosition = stagedDefaultPosition; - this.resetProgress = resetProgress; } + LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", + this.id, jobProperties, dataSourceProperties); } @Override @@ -847,7 +773,6 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); setRoutineLoadDesc(log.getRoutineLoadDesc()); - updateLoadDefinition(log.getDataSourceProperties()); } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 462803af886af1..ca0748775e0f28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -29,11 +29,10 @@ import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; -import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; @@ -50,15 +49,12 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRoutineLoadTask; import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import org.apache.kafka.common.PartitionInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -79,7 +75,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; public class KafkaRoutineLoadJobTest { @@ -287,182 +282,71 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro } @Test - public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job_atomic", 1L, - 1L, "127.0.0.1:9020", "topic-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map originalCustomProperties = Maps.newHashMap(); - originalCustomProperties.put("client.id", "old-client"); - Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); - Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); - Map originalProgress = Maps.newHashMap(); - originalProgress.put(0, 10L); - Deencapsulation.setField(routineLoadJob, "progress", new KafkaProgress(originalProgress)); - routineLoadJob.updateLoadDefinition(null); - Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); - - Map originalDataSourceProperties = Maps.newHashMap(); - originalDataSourceProperties.put("property.client.id", "new-client"); - KafkaDataSourceProperties dataSourceProperties = - new KafkaDataSourceProperties(originalDataSourceProperties); - Map alteredCustomProperties = Maps.newHashMap(); - alteredCustomProperties.put("client.id", "new-client"); - Deencapsulation.setField(dataSourceProperties, "customKafkaProperties", alteredCustomProperties); - dataSourceProperties.setKafkaPartitionOffsets(Lists.newArrayList(Pair.of(1, 20L))); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); - Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - - try (MockedStatic kafkaUtilStatic = Mockito.mockStatic(KafkaUtil.class)) { - kafkaUtilStatic.when(() -> KafkaUtil.getRealOffsets( - Mockito.eq("127.0.0.1:9020"), Mockito.eq("topic-1"), Mockito.anyMap(), Mockito.anyList(), - Mockito.nullable(String.class))) - .thenReturn(Lists.newArrayList(Pair.of(1, 20L))); - Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); - } - - Assert.assertEquals("topic-1", routineLoadJob.getTopic()); - Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); - Map currentConvertedProperties = - Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); - Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); - Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); - Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); - } - - @Test - public void testSuccessfulAlterUpdatesLoadDefinitionAndJournal() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - routineLoadJob.updateLoadDefinition(null); - Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); + public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exception { + KafkaRoutineLoadJob leader = createPausedJob(); + KafkaRoutineLoadJob follower = createPausedJob(); + RoutineLoadDesc originalDesc = new RoutineLoadDesc(new Separator("|", "|"), null, null, + null, null, null, null, LoadTask.MergeType.APPEND, "original_sequence"); + leader.setRoutineLoadDesc(originalDesc); + follower.setRoutineLoadDesc(originalDesc); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, - LoadTask.MergeType.APPEND, "sequence_col"); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), null, + null, null, null, null, LoadTask.MergeType.APPEND, null); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(null); - Mockito.when(command.getRoutineLoadDesc()).thenReturn(routineLoadDesc); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); + Env env = Mockito.mock(Env.class); EditLog editLog = Mockito.mock(EditLog.class); + AlterRoutineLoadJobOperationLog alterLog; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Mockito.when(env.getEditLog()).thenReturn(editLog); - routineLoadJob.modifyProperties(command); + leader.modifyProperties(command); ArgumentCaptor logCaptor = ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); - Assert.assertSame(routineLoadDesc, logCaptor.getValue().getRoutineLoadDesc()); + alterLog = logCaptor.getValue(); } - Assert.assertEquals("sequence_col", routineLoadJob.getSequenceCol()); - Assert.assertNotSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); - } - - @Test - public void testImageRoundTripUsesPersistedLoadDefinitionAfterAlterReplay() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, - 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Deencapsulation.setField(routineLoadJob, "desireTaskConcurrentNum", 1); - String originalCreateSql = "CREATE ROUTINE LOAD db1.job1 ON table1 " - + "COLUMNS TERMINATED BY '|' " - + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; - Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(originalCreateSql, 0)); - routineLoadJob.updateLoadDefinition(null); - - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( - new Separator(",", ","), null, null, null, null, null, null, - LoadTask.MergeType.APPEND, "sequence_col"); - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "2"); - routineLoadJob.replayModifyProperties(new AlterRoutineLoadJobOperationLog( - routineLoadJob.getId(), jobProperties, null, routineLoadDesc)); + Assert.assertSame(delta, alterLog.getRoutineLoadDesc()); + Assert.assertEquals(jobProperties, alterLog.getJobProperties()); + assertAlterState(leader); - Env env = Mockito.mock(Env.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); - connectContextStatic.close(); - connectContextStatic = null; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("db1"); - Mockito.when(database.getFullName()).thenReturn("db1"); - Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); - Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("table1"); - Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + follower.replayModifyProperties(alterLog); + assertAlterState(follower); - RoutineLoadJob restored = imageRoundTrip(routineLoadJob); - - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals("sequence_col", restored.getSequenceCol()); - Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); - Assert.assertEquals(2, (int) Deencapsulation.getField(restored, "desireTaskConcurrentNum")); - Assert.assertNotNull(Deencapsulation.getField(restored, "loadDefinition")); - - // Simulate an older FE ignoring the unknown loadDefinition field. It can read the image - // through origStmt, but the ALTERed load clauses are intentionally not preserved on rollback. - RoutineLoadJob rollbackRestored = imageRoundTripWithoutLoadDefinition(routineLoadJob); - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, rollbackRestored.getState()); - Assert.assertEquals("|", rollbackRestored.getColumnSeparator().getSeparator()); - Assert.assertNull(rollbackRestored.getSequenceCol()); - Assert.assertNull(Deencapsulation.getField(rollbackRestored, "loadDefinition")); - } + assertAlterState(imageRoundTrip(leader)); + assertAlterState(imageRoundTrip(follower)); } - @Test - public void testImageRoundTripRestoresLegacyOrigStmt() throws Exception { - KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "job1", 1L, + private static KafkaRoutineLoadJob createPausedJob() { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1L, "job1", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - String createSql = "CREATE ROUTINE LOAD db1.job1 ON stale_table " - + "COLUMNS TERMINATED BY ',' " - + "FROM KAFKA ('kafka_broker_list' = '127.0.0.1:9020', 'kafka_topic' = 'topic1')"; - Deencapsulation.setField(routineLoadJob, "origStmt", new OriginStatement(createSql, 0)); - Deencapsulation.setField(routineLoadJob, "loadDefinition", null); - - Env env = Mockito.mock(Env.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); - connectContextStatic.close(); - connectContextStatic = null; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("db1"); - Mockito.when(database.getTable(1L)).thenReturn(Optional.of((Table) table)); - Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("table1"); - Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); - - RoutineLoadJob restored = imageRoundTrip(routineLoadJob); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.PAUSED); + return job; + } - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals(",", restored.getColumnSeparator().getSeparator()); - Assert.assertNull(Deencapsulation.getField(restored, "loadDefinition")); - } + private static void assertAlterState(RoutineLoadJob job) { + Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); + Assert.assertEquals("original_sequence", job.getSequenceCol()); + Assert.assertEquals((byte) '"', job.getEnclose()); + Assert.assertEquals((byte) '\\', job.getEscape()); + Assert.assertTrue(job.getEmptyFieldAsNull()); + Assert.assertEquals(Boolean.TRUE, Deencapsulation.getField(job, "emptyFieldAsNull")); + + Map persistedJobProperties = Deencapsulation.getField(job, "jobProperties"); + Assert.assertEquals("\"", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + Assert.assertEquals("\\", persistedJobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); + Assert.assertEquals("true", persistedJobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) throws Exception { @@ -475,29 +359,6 @@ private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) thro } } - private static RoutineLoadJob imageRoundTripWithoutLoadDefinition(RoutineLoadJob routineLoadJob) - throws Exception { - ByteArrayOutputStream image = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(image)) { - routineLoadJob.write(out); - } - - String json; - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image.toByteArray()))) { - json = Text.readString(in); - } - JsonObject jobJson = JsonParser.parseString(json).getAsJsonObject(); - Assert.assertNotNull(jobJson.remove("ld")); - - ByteArrayOutputStream legacyImage = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(legacyImage)) { - Text.writeString(out, jobJson.toString()); - } - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(legacyImage.toByteArray()))) { - return RoutineLoadJob.read(in); - } - } - @Test public void testDisplayCustomPropertiesMasksKafkaSecrets() { KafkaRoutineLoadJob routineLoadJob = new KafkaRoutineLoadJob(1L, "kafka_routine_load_job", 1L, diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index a9face05891ab1..ea64e7e2871b0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -17,25 +17,39 @@ package org.apache.doris.load.routineload; +import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; -import org.apache.doris.common.Pair; -import org.apache.doris.common.UserException; +import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; import org.apache.doris.load.routineload.kinesis.KinesisDataSourceProperties; import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; +import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; +import org.apache.doris.persist.EditLog; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.Gson; +import com.google.gson.JsonParser; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -234,47 +248,55 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi } @Test - public void testFailedAlterDoesNotChangeRuntimeOrLoadDefinition() throws Exception { - KinesisRoutineLoadJob routineLoadJob = new KinesisRoutineLoadJob(1L, "job_atomic", 1L, - 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); - Deencapsulation.setField(routineLoadJob, "state", RoutineLoadJob.JobState.PAUSED); - Map originalCustomProperties = Maps.newHashMap(); - originalCustomProperties.put("client.id", "old-client"); - Deencapsulation.setField(routineLoadJob, "customProperties", originalCustomProperties); - Map originalConvertedProperties = Maps.newHashMap(originalCustomProperties); - Deencapsulation.setField(routineLoadJob, "convertedCustomProperties", originalConvertedProperties); - Deencapsulation.setField(routineLoadJob, "customKinesisShards", Lists.newArrayList("shard-1")); - Map originalProgress = Maps.newHashMap(); - originalProgress.put("shard-1", "10"); - Deencapsulation.setField(routineLoadJob, "progress", new KinesisProgress(originalProgress)); - routineLoadJob.updateLoadDefinition(null); - Object originalLoadDefinition = Deencapsulation.getField(routineLoadJob, "loadDefinition"); - - Map originalDataSourceProperties = Maps.newHashMap(); - originalDataSourceProperties.put("property.client.id", "new-client"); - originalDataSourceProperties.put(KinesisConfiguration.KINESIS_REGION.getName(), "us-east-1"); - KinesisDataSourceProperties dataSourceProperties = - new KinesisDataSourceProperties(originalDataSourceProperties); - Map alteredCustomProperties = Maps.newHashMap(); - alteredCustomProperties.put("client.id", "new-client"); - Deencapsulation.setField(dataSourceProperties, "customKinesisProperties", alteredCustomProperties); - Deencapsulation.setField(dataSourceProperties, "region", "us-east-1"); - dataSourceProperties.setKinesisShardPositions(Lists.newArrayList(Pair.of("shard-2", "20"))); + public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exception { + KinesisRoutineLoadJob leader = createPausedJobWithInitialLoadDesc(); + KinesisRoutineLoadJob replay = createPausedJobWithInitialLoadDesc(); + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), + null, null, null, null, null, LoadTask.MergeType.APPEND, "sequence_col"); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(Maps.newHashMap()); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(null); + Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); + + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + ArgumentCaptor logCaptor = + ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getEditLog()).thenReturn(editLog); + leader.modifyProperties(command); + Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + } + + AlterRoutineLoadJobOperationLog log = logCaptor.getValue(); + replay.replayModifyProperties(log); + + Assert.assertSame(delta, log.getRoutineLoadDesc()); + assertAlterResult(leader); + assertAlterResult(replay); + Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), + JsonParser.parseString(checkpointJson(replay))); + } + + @Test + public void testAlterValidatesCsvBeforeDataSourceMutation() { + KinesisRoutineLoadJob job = createPausedJobWithInitialLoadDesc(); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); + KinesisDataSourceProperties dataSourceProperties = Mockito.mock(KinesisDataSourceProperties.class); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - Assert.assertThrows(UserException.class, () -> routineLoadJob.modifyProperties(command)); - - Assert.assertEquals("ap-southeast-1", routineLoadJob.getRegion()); - Assert.assertEquals(Lists.newArrayList("shard-1"), - Deencapsulation.getField(routineLoadJob, "customKinesisShards")); - Map currentCustomProperties = Deencapsulation.getField(routineLoadJob, "customProperties"); - Map currentConvertedProperties = - Deencapsulation.getField(routineLoadJob, "convertedCustomProperties"); - Assert.assertEquals("old-client", currentCustomProperties.get("client.id")); - Assert.assertEquals("old-client", currentConvertedProperties.get("client.id")); - Assert.assertSame(originalLoadDefinition, Deencapsulation.getField(routineLoadJob, "loadDefinition")); + Assert.assertThrows(AnalysisException.class, () -> job.modifyProperties(command)); + Assert.assertEquals("stream-1", job.getStream()); + Mockito.verifyNoInteractions(dataSourceProperties); } @Test @@ -387,6 +409,35 @@ public void testDisplayCustomPropertiesMasksKinesisSecrets() { Assert.assertEquals("role_arn_value", showCreateCustomProperties.get("property.aws.role_arn")); } + private KinesisRoutineLoadJob createPausedJobWithInitialLoadDesc() { + KinesisRoutineLoadJob job = new KinesisRoutineLoadJob(1L, "kinesis_routine_load_job", 1L, + 1L, "ap-southeast-1", "stream-1", UserIdentity.ADMIN); + Deencapsulation.setField(job, "state", RoutineLoadJob.JobState.PAUSED); + Deencapsulation.setField(job, "createTimestamp", 123L); + job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator("|", "|"), null, + null, null, null, null, null, LoadTask.MergeType.APPEND, null)); + return job; + } + + private void assertAlterResult(KinesisRoutineLoadJob job) { + Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); + Assert.assertEquals("sequence_col", job.getSequenceCol()); + Assert.assertEquals((byte) '"', job.getEnclose()); + Assert.assertEquals((byte) '\\', job.getEscape()); + Assert.assertTrue(job.getEmptyFieldAsNull()); + } + + private String checkpointJson(RoutineLoadJob job) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + job.write(out); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return Text.readString(in); + } + } + private Set collectAssignedShards(KinesisRoutineLoadJob routineLoadJob) { List routineLoadTaskInfoList = Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java new file mode 100644 index 00000000000000..7da9e1d3308589 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -0,0 +1,426 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.load.routineload; + +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.info.PartitionNamesInfo; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.load.routineload.kafka.KafkaConfiguration; +import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; +import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; +import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class RoutineLoadJobPersistenceTest { + private static final String LEGACY_IMAGE = + "/upgrade/routine-load/a8928245/routine-load-kafka-image.b64"; + + @Test + public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "direct_job", 1002L, + 1003L, "127.0.0.1:9092", "direct_topic", UserIdentity.ADMIN); + job.state = RoutineLoadJob.JobState.PAUSED; + job.origStmt = new OriginStatement("this is deliberately not valid SQL", 0); + + Separator columnSeparator = analyzedSeparator("\\x01"); + Separator lineDelimiter = analyzedSeparator("\\n"); + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new IntLiteral(7L))); + Expr precedingFilter = predicate(BinaryPredicate.Operator.GT, "source_col", 1L); + Expr whereExpr = predicate(BinaryPredicate.Operator.LE, "mapped_col", 10L); + Expr deleteCondition = predicate(BinaryPredicate.Operator.EQ, "delete_flag", 1L); + PartitionNamesInfo partitions = new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")); + job.setRoutineLoadDesc(new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, whereExpr, partitions, deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); + + job.desireTaskConcurrentNum = 5; + job.maxErrorNum = 17L; + job.maxBatchIntervalS = 23L; + job.maxBatchRows = 300001L; + job.maxBatchSizeBytes = 104857601L; + job.execMemLimit = 345678901L; + job.maxFilterRatio = 0.99; + job.sendBatchParallelism = 99; + job.loadToSingleTablet = false; + job.memtableOnSinkNode = true; + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, "0.25"); + jobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, "4"); + jobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, "true"); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FIXED_COLUMNS"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, "ERROR"); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + job.jobProperties = jobProperties; + + JsonObject json = imageJson(job); + Assert.assertEquals(1, json.get("rlpv").getAsInt()); + for (String key : Lists.newArrayList( + "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc", "eml", "mosn")) { + Assert.assertTrue("missing direct-state key " + key, json.has(key)); + } + Assert.assertFalse(json.has("ld")); + Assert.assertEquals("\\x01", json.getAsJsonObject("cs").get("os").getAsString()); + Assert.assertEquals("\u0001", json.getAsJsonObject("cs").get("s").getAsString()); + Assert.assertEquals("\\n", json.getAsJsonObject("lidel").get("os").getAsString()); + Assert.assertEquals("\n", json.getAsJsonObject("lidel").get("s").getAsString()); + Assert.assertEquals(2, json.getAsJsonObject("cds").getAsJsonArray("des").size()); + + RoutineLoadJob restored; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restored = imageRoundTrip(job); + envStatic.verifyNoInteractions(); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), restored.getPartitionNamesInfo().getPartitionNames()); + Assert.assertEquals(2, restored.columnDescs.descs.size()); + Assert.assertEquals("source_col", restored.columnDescs.descs.get(0).getColumnName()); + Assert.assertEquals("mapped_col", restored.columnDescs.descs.get(1).getColumnName()); + Assert.assertNotNull(restored.columnDescs.descs.get(1).getExpr()); + Assert.assertNotNull(restored.getPrecedingFilter()); + Assert.assertNotNull(restored.getWhereExpr()); + Assert.assertEquals("\\x01", restored.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("\u0001", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals("\\n", restored.getLineDelimiter().getOriSeparator()); + Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); + Assert.assertEquals("seq_col", restored.getSequenceCol()); + Assert.assertEquals(LoadTask.MergeType.MERGE, restored.getMergeType()); + Assert.assertNotNull(restored.getDeleteCondition()); + Assert.assertEquals(345678901L, restored.getMemLimit()); + Assert.assertTrue(restored.isMemtableOnSinkNode()); + Assert.assertEquals(5, restored.desireTaskConcurrentNum); + Assert.assertEquals(17L, restored.maxErrorNum); + Assert.assertEquals(23L, restored.getMaxBatchIntervalS()); + Assert.assertEquals(300001L, restored.getMaxBatchRows()); + Assert.assertEquals(104857601L, restored.getMaxBatchSizeBytes()); + + NereidsRoutineLoadTaskInfo taskInfo = restored.toNereidsRoutineLoadTaskInfo(); + Assert.assertEquals(345678901L, taskInfo.getMemLimit()); + Assert.assertEquals(0.25, taskInfo.getMaxFilterRatio(), 0.0); + Assert.assertEquals(4, taskInfo.getSendBatchParallelism()); + Assert.assertTrue(taskInfo.isLoadToSingleTablet()); + Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, taskInfo.getUniqueKeyUpdateMode()); + Assert.assertTrue(taskInfo.isFixedPartialUpdate()); + Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, taskInfo.getPartialUpdateNewRowPolicy()); + Assert.assertEquals((byte) '"', taskInfo.getEnclose()); + Assert.assertEquals((byte) '\\', taskInfo.getEscape()); + Assert.assertTrue(taskInfo.getEmptyFieldAsNull()); + Assert.assertTrue(taskInfo.isMemtableOnSinkNode()); + Assert.assertEquals(LoadTask.MergeType.MERGE, taskInfo.getMergeType()); + Assert.assertNotNull(taskInfo.getDeleteCondition()); + Assert.assertEquals("seq_col", taskInfo.getSequenceCol()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + taskInfo.getPartitionNamesInfo().getPartitionNames()); + Assert.assertEquals(2, taskInfo.getColumnExprDescs().descs.size()); + Assert.assertNotNull(taskInfo.getPrecedingFilter()); + Assert.assertNotNull(taskInfo.getWhereExpr()); + Assert.assertEquals("\u0001", taskInfo.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", taskInfo.getLineDelimiter().getSeparator()); + } + + @Test + public void testDirectStateImageWithNoLoadClausesDoesNotFallback() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "empty_job", 2002L, + 2003L, "127.0.0.1:9092", "empty_topic", UserIdentity.ADMIN); + job.state = RoutineLoadJob.JobState.PAUSED; + job.origStmt = new OriginStatement("also not valid SQL", 0); + + JsonObject json = imageJson(job); + Assert.assertEquals(1, json.get("rlpv").getAsInt()); + for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "lidel", "sc", "dc")) { + Assert.assertFalse("unexpected nullable direct-state key " + key, json.has(key)); + } + + RoutineLoadJob restored; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restored = imageRoundTrip(job); + envStatic.verifyNoInteractions(); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertNull(restored.getPartitionNamesInfo()); + Assert.assertNull(restored.columnDescs); + Assert.assertNull(restored.getPrecedingFilter()); + Assert.assertNull(restored.getWhereExpr()); + Assert.assertNull(restored.getColumnSeparator()); + Assert.assertNull(restored.getLineDelimiter()); + Assert.assertNull(restored.getSequenceCol()); + Assert.assertNull(restored.getDeleteCondition()); + Assert.assertEquals(LoadTask.MergeType.APPEND, restored.getMergeType()); + } + + @Test + public void testLegacyImageMigratesOnce() throws Exception { + Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(8001L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("legacy_db")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrAnalysisException("legacy_db")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("legacy_db"); + Mockito.when(database.getTable(9001L)).thenReturn(Optional.of((Table) table)); + Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("current_table"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(table.getEnableUniqueKeyMergeOnWrite()).thenReturn(true); + + RoutineLoadJob migrated; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + migrated = readImage(loadBase64Fixture(LEGACY_IMAGE)); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, migrated.getState()); + Assert.assertEquals("|", migrated.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("|", migrated.getColumnSeparator().getSeparator()); + Assert.assertNull(migrated.getSequenceCol()); + Assert.assertEquals(33554432L, migrated.getMemLimit()); + Assert.assertEquals(0.25, migrated.getMaxFilterRatio(), 0.0); + Assert.assertEquals(3, migrated.getSendBatchParallelism()); + Assert.assertTrue(migrated.isLoadToSingleTablet()); + Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, migrated.getUniqueKeyUpdateMode()); + Assert.assertTrue(migrated.isFixedPartialUpdate()); + Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, migrated.partialUpdateNewKeyPolicy); + Assert.assertEquals((byte) '"', migrated.getEnclose()); + Assert.assertEquals((byte) '\\', migrated.getEscape()); + Assert.assertTrue(migrated.getEmptyFieldAsNull()); + Assert.assertFalse(migrated.isMemtableOnSinkNode()); + + JsonObject migratedJson = imageJson(migrated); + Assert.assertEquals(1, migratedJson.get("rlpv").getAsInt()); + Assert.assertTrue(migratedJson.has("cs")); + Assert.assertTrue(migratedJson.has("eml")); + Assert.assertTrue(migratedJson.has("mosn")); + migrated.origStmt = new OriginStatement("invalid after successful migration", 0); + + RoutineLoadJob restoredAgain; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restoredAgain = imageRoundTrip(migrated); + envStatic.verifyNoInteractions(); + } + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restoredAgain.getState()); + Assert.assertEquals("|", restoredAgain.getColumnSeparator().getSeparator()); + Assert.assertEquals(33554432L, restoredAgain.getMemLimit()); + Assert.assertFalse(restoredAgain.isMemtableOnSinkNode()); + } + + @Test + public void testKafkaDerivedStateIsRebuiltFromDurableProperties() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(3001L, "kafka_derived", 3002L, + 3003L, "127.0.0.1:9092", "derived_topic", UserIdentity.ADMIN); + job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); + Map customProperties = Maps.newHashMap(); + customProperties.put("client.id", "durable-client"); + customProperties.put(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName(), "OFFSET_BEGINNING"); + Deencapsulation.setField(job, "customProperties", customProperties); + Deencapsulation.setField(job, "customKafkaPartitions", Lists.newArrayList(9)); + Deencapsulation.setField(job, "currentKafkaPartitions", Lists.newArrayList(1, 2)); + Deencapsulation.setField(job, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("stale", "value"))); + Deencapsulation.setField(job, "cachedPartitionWithLatestOffsets", + Maps.newHashMap(ImmutableMap.of(1, 100L))); + Deencapsulation.setField(job, "newCurrentKafkaPartition", Lists.newArrayList(3)); + Deencapsulation.setField(job, "kafkaDefaultOffSet", "OFFSET_END"); + + JsonObject json = imageJson(job); + Assert.assertEquals("127.0.0.1:9092", json.get("bl").getAsString()); + Assert.assertEquals("derived_topic", json.get("tp").getAsString()); + Assert.assertEquals("durable-client", json.getAsJsonObject("prop").get("client.id").getAsString()); + Assert.assertEquals(1, json.getAsJsonArray("cskp").size()); + assertNoJavaFieldNames(json, "currentKafkaPartitions", "convertedCustomProperties", + "cachedPartitionWithLatestOffsets", "newCurrentKafkaPartition", "kafkaDefaultOffSet"); + + KafkaRoutineLoadJob restored = (KafkaRoutineLoadJob) imageRoundTrip(job); + Assert.assertEquals("127.0.0.1:9092", restored.getBrokerList()); + Assert.assertEquals("derived_topic", restored.getTopic()); + Assert.assertEquals(Lists.newArrayList(9), Deencapsulation.getField(restored, "customKafkaPartitions")); + Assert.assertTrue(((List) Deencapsulation.getField(restored, "currentKafkaPartitions")).isEmpty()); + Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); + Assert.assertTrue(((Map) Deencapsulation.getField( + restored, "cachedPartitionWithLatestOffsets")).isEmpty()); + Assert.assertEquals("", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + + Env env = Mockito.mock(Env.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + restored.prepare(); + } + Assert.assertEquals("durable-client", restored.getConvertedCustomProperties().get("client.id")); + Assert.assertFalse(restored.getConvertedCustomProperties().containsKey( + KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); + Assert.assertEquals("OFFSET_BEGINNING", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + } + + @Test + public void testKinesisDerivedStateIsRebuiltFromDurableProperties() throws Exception { + KinesisRoutineLoadJob job = new KinesisRoutineLoadJob(4001L, "kinesis_derived", 4002L, + 4003L, "us-east-1", "derived_stream", UserIdentity.ADMIN); + job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); + Deencapsulation.setField(job, "endpoint", "https://kinesis.example.test"); + Map customProperties = Maps.newHashMap(); + customProperties.put("client.setting", "durable-value"); + customProperties.put("kinesis_default_pos", "TRIM_HORIZON"); + Deencapsulation.setField(job, "customProperties", customProperties); + Deencapsulation.setField(job, "customKinesisShards", Lists.newArrayList("custom-shard")); + Deencapsulation.setField(job, "openKinesisShards", Lists.newArrayList("open-shard")); + Deencapsulation.setField(job, "closedKinesisShards", Lists.newArrayList("closed-shard")); + Deencapsulation.setField(job, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("stale", "value"))); + Deencapsulation.setField(job, "cachedShardWithMillsBehindLatest", + Maps.newHashMap(ImmutableMap.of("open-shard", 99L))); + Deencapsulation.setField(job, "newCurrentKinesisShards", Lists.newArrayList("new-shard")); + Deencapsulation.setField(job, "kinesisDefaultPosition", "LATEST"); + + JsonObject json = imageJson(job); + Assert.assertEquals("us-east-1", json.get("rg").getAsString()); + Assert.assertEquals("derived_stream", json.get("stm").getAsString()); + Assert.assertEquals("https://kinesis.example.test", json.get("ep").getAsString()); + Assert.assertEquals("durable-value", + json.getAsJsonObject("prop").get("client.setting").getAsString()); + Assert.assertEquals("custom-shard", json.getAsJsonArray("csks").get(0).getAsString()); + Assert.assertEquals("open-shard", json.getAsJsonArray("opks").get(0).getAsString()); + Assert.assertEquals("closed-shard", json.getAsJsonArray("clks").get(0).getAsString()); + assertNoJavaFieldNames(json, "convertedCustomProperties", "cachedShardWithMillsBehindLatest", + "newCurrentKinesisShards", "kinesisDefaultPosition"); + + KinesisRoutineLoadJob restored = (KinesisRoutineLoadJob) imageRoundTrip(job); + Assert.assertEquals("us-east-1", restored.getRegion()); + Assert.assertEquals("derived_stream", restored.getStream()); + Assert.assertEquals("https://kinesis.example.test", restored.getEndpoint()); + Assert.assertEquals(Lists.newArrayList("custom-shard"), + Deencapsulation.getField(restored, "customKinesisShards")); + Assert.assertEquals(Lists.newArrayList("open-shard"), + Deencapsulation.getField(restored, "openKinesisShards")); + Assert.assertEquals(Lists.newArrayList("closed-shard"), + Deencapsulation.getField(restored, "closedKinesisShards")); + Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); + Assert.assertTrue(((Map) Deencapsulation.getField( + restored, "cachedShardWithMillsBehindLatest")).isEmpty()); + Assert.assertTrue(((List) Deencapsulation.getField(restored, "newCurrentKinesisShards")).isEmpty()); + Assert.assertEquals("", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + + restored.prepare(); + Assert.assertEquals("durable-value", restored.getConvertedCustomProperties().get("client.setting")); + Assert.assertEquals("TRIM_HORIZON", + restored.getConvertedCustomProperties().get("kinesis_default_pos")); + Assert.assertEquals("TRIM_HORIZON", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + } + + private static Separator analyzedSeparator(String value) throws Exception { + Separator separator = new Separator(value); + separator.analyze(); + return separator; + } + + private static Expr predicate(BinaryPredicate.Operator operator, String column, long value) { + return new BinaryPredicate(operator, new SlotRef(null, column), new IntLiteral(value)); + } + + private static JsonObject imageJson(RoutineLoadJob job) throws IOException { + byte[] image = writeImage(job); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image))) { + return JsonParser.parseString(Text.readString(in)).getAsJsonObject(); + } + } + + private static RoutineLoadJob imageRoundTrip(RoutineLoadJob job) throws IOException { + return readImage(writeImage(job)); + } + + private static byte[] writeImage(RoutineLoadJob job) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + job.write(out); + } + return bytes.toByteArray(); + } + + private static RoutineLoadJob readImage(byte[] image) throws IOException { + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image))) { + return RoutineLoadJob.read(in); + } + } + + private static byte[] loadBase64Fixture(String resource) throws IOException { + try (InputStream in = RoutineLoadJobPersistenceTest.class.getResourceAsStream(resource)) { + if (in == null) { + throw new IOException("missing fixture " + resource); + } + String base64 = new String(in.readAllBytes(), StandardCharsets.UTF_8).trim(); + return Base64.getDecoder().decode(base64); + } + } + + private static void assertNoJavaFieldNames(JsonObject json, String... fieldNames) { + for (String fieldName : fieldNames) { + Assert.assertFalse("derived field leaked into image: " + fieldName, json.has(fieldName)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 522c8bee4d9948..058cf42f949d54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -17,15 +17,22 @@ package org.apache.doris.persist; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.UserException; -import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.gson.GsonUtils; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -34,23 +41,19 @@ import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; import java.util.Map; public class AlterRoutineLoadOperationLogTest { - private static String fileName = "./AlterRoutineLoadOperationLogTest"; + private static final String A8928245_LEGACY_LOG = + "/upgrade/routine-load/a8928245/alter-routine-load-log.b64"; @Test public void testSerializeAlterRoutineLoadOperationLog() throws IOException, UserException { - // 1. Write objects to file - File file = new File(fileName); - file.createNewFile(); - file.deleteOnExit(); - DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); - long jobId = 1000; Map jobProperties = Maps.newHashMap(); jobProperties.put(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY, "5"); @@ -65,18 +68,31 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(null, null, null, null, null, null, null, - LoadTask.MergeType.APPEND, "sequence_col"); + Separator columnSeparator = new Separator(",", "\\x2c"); + Separator lineDelimiter = new Separator("\n", "\\n"); + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new StringLiteral("mapped_value"))); + BinaryPredicate precedingFilter = new BinaryPredicate(BinaryPredicate.Operator.GT, + new IntLiteral(3L), new IntLiteral(2L)); + BinaryPredicate where = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new StringLiteral("selected"), new StringLiteral("selected")); + PartitionNamesInfo partitions = new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")); + BinaryPredicate deleteCondition = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new IntLiteral(1L), new IntLiteral(1L)); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, where, partitions, deleteCondition, LoadTask.MergeType.MERGE, "sequence_col"); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, jobProperties, routineLoadDataSourceProperties, routineLoadDesc); - log.write(out); - out.flush(); - out.close(); - - // 2. Read objects from file - DataInputStream in = new DataInputStream(new FileInputStream(file)); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + log.write(out); + } - AlterRoutineLoadJobOperationLog log2 = AlterRoutineLoadJobOperationLog.read(in); + AlterRoutineLoadJobOperationLog log2; + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + log2 = AlterRoutineLoadJobOperationLog.read(in); + } Assert.assertEquals(1, log2.getJobProperties().size()); Assert.assertEquals("5", log2.getJobProperties().get(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); KafkaDataSourceProperties kafkaDataSourceProperties = (KafkaDataSourceProperties) log2.getDataSourceProperties(); @@ -88,24 +104,46 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); - Assert.assertEquals("sequence_col", log2.getRoutineLoadDesc().getSequenceColName()); - - in.close(); + RoutineLoadDesc restoredDesc = log2.getRoutineLoadDesc(); + Assert.assertEquals(",", restoredDesc.getColumnSeparator().getSeparator()); + Assert.assertEquals("\\x2c", restoredDesc.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("\n", restoredDesc.getLineDelimiter().getSeparator()); + Assert.assertEquals("\\n", restoredDesc.getLineDelimiter().getOriSeparator()); + Assert.assertEquals(2, restoredDesc.getColumnsInfo().size()); + Assert.assertEquals("source_col", restoredDesc.getColumnsInfo().get(0).getColumnName()); + Assert.assertEquals("mapped_col", restoredDesc.getColumnsInfo().get(1).getColumnName()); + Assert.assertNotNull(restoredDesc.getColumnsInfo().get(1).getExpr()); + Assert.assertNotNull(restoredDesc.getPrecedingFilter()); + Assert.assertNotNull(restoredDesc.getFilter()); + Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + restoredDesc.getPartitionNamesInfo().getPartitionNames()); + Assert.assertNotNull(restoredDesc.getDeleteCondition()); + Assert.assertEquals(LoadTask.MergeType.MERGE, restoredDesc.getMergeType()); + Assert.assertEquals("sequence_col", restoredDesc.getSequenceColName()); + Assert.assertEquals(GsonUtils.GSON.toJson(routineLoadDesc), GsonUtils.GSON.toJson(restoredDesc)); } @Test public void testDeserializeLegacyLogWithoutRoutineLoadDesc() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(bytes)) { - Text.writeString(out, "{\"jobId\":1000,\"jobProperties\":{}," - + "\"dataSourceProperties\":null}"); - } - - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + byte[] bytes = loadBase64Fixture(A8928245_LEGACY_LOG); + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes))) { AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); - Assert.assertEquals(1000L, log.getJobId()); + Assert.assertEquals(7001L, log.getJobId()); + Assert.assertTrue(log.getJobProperties().isEmpty()); + Assert.assertNull(log.getDataSourceProperties()); Assert.assertNull(log.getRoutineLoadDesc()); } } + private static byte[] loadBase64Fixture(String resource) throws IOException { + try (InputStream in = AlterRoutineLoadOperationLogTest.class.getResourceAsStream(resource)) { + if (in == null) { + throw new IOException("missing fixture " + resource); + } + String base64 = new String(in.readAllBytes(), StandardCharsets.UTF_8).trim(); + return Base64.getDecoder().decode(base64); + } + } + } From 419ac458fcbce0e374f3cc6e8999ab11b07d688a Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 17 Aug 2026 00:34:56 +0800 Subject: [PATCH 04/11] temp --- .../load/routineload/RoutineLoadJob.java | 20 ++++++++---------- .../RoutineLoadJobPersistenceTest.java | 21 ++++++++++++++----- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 57497876059a81..c01edd1ddcd28b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -116,8 +116,6 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); - private static final int CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION = 1; - public static final long DEFAULT_MAX_ERROR_NUM = 0; public static final double DEFAULT_MAX_FILTER_RATIO = 1.0; @@ -180,9 +178,6 @@ public boolean isFinalState() { protected long dbId; @SerializedName("tbid") protected long tableId; - // An absent version identifies a legacy record whose CREATE statement still needs to be migrated. - @SerializedName("rlpv") - private int routineLoadPersistenceVersion; // this code is used to verify be task request protected long authCode; // protected RoutineLoadDesc routineLoadDesc; // optional @@ -283,7 +278,7 @@ public boolean isFinalState() { protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @SerializedName("mt") - protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete + protected LoadTask.MergeType mergeType; @SerializedName("dc") protected Expr deleteCondition; // TODO(ml): error sample @@ -330,7 +325,7 @@ public RoutineLoadJob(Long id, String name, this.tableId = tableId; this.authCode = 0; this.userIdentity = userIdentity; - this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; + this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -360,7 +355,7 @@ public RoutineLoadJob(Long id, String name, this.authCode = 0; this.userIdentity = userIdentity; this.isMultiTable = true; - this.routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; + this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -1983,16 +1978,19 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } - if (routineLoadPersistenceVersion == 0) { + // Legacy images did not persist mergeType. New images always contain it, including jobs + // without any load clause, so its absence is sufficient to identify the one-time fallback. + boolean isOldImage = mergeType == null; + if (isOldImage) { + mergeType = LoadTask.MergeType.APPEND; // Legacy images did not persist this create-time session option. Preserve their historical // post-restart behavior instead of inheriting the image-loading thread's ConnectContext. memtableOnSinkNode = false; } try { hydrateJobProperties(); - if (routineLoadPersistenceVersion == 0) { + if (isOldImage) { restoreLegacyDefinition(); - routineLoadPersistenceVersion = CURRENT_ROUTINE_LOAD_PERSISTENCE_VERSION; } } catch (Exception e) { this.state = JobState.CANCELLED; diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 7da9e1d3308589..5c221334e22254 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -114,7 +114,8 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception job.jobProperties = jobProperties; JsonObject json = imageJson(job); - Assert.assertEquals(1, json.get("rlpv").getAsInt()); + Assert.assertTrue(json.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.MERGE.name(), json.get("mt").getAsString()); for (String key : Lists.newArrayList( "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc", "eml", "mosn")) { Assert.assertTrue("missing direct-state key " + key, json.has(key)); @@ -187,7 +188,8 @@ public void testDirectStateImageWithNoLoadClausesDoesNotFallback() throws Except job.origStmt = new OriginStatement("also not valid SQL", 0); JsonObject json = imageJson(job); - Assert.assertEquals(1, json.get("rlpv").getAsInt()); + Assert.assertTrue(json.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.APPEND.name(), json.get("mt").getAsString()); for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "lidel", "sc", "dc")) { Assert.assertFalse("unexpected nullable direct-state key " + key, json.has(key)); } @@ -230,11 +232,16 @@ public void testLegacyImageMigratesOnce() throws Exception { Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); Mockito.when(table.getEnableUniqueKeyMergeOnWrite()).thenReturn(true); + byte[] legacyImage = loadBase64Fixture(LEGACY_IMAGE); + JsonObject legacyJson = imageJson(legacyImage); + Assert.assertFalse(legacyJson.has("mt")); + Assert.assertTrue(legacyJson.has("ostmt")); + RoutineLoadJob migrated; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - migrated = readImage(loadBase64Fixture(LEGACY_IMAGE)); + migrated = readImage(legacyImage); } Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, migrated.getState()); @@ -254,7 +261,8 @@ public void testLegacyImageMigratesOnce() throws Exception { Assert.assertFalse(migrated.isMemtableOnSinkNode()); JsonObject migratedJson = imageJson(migrated); - Assert.assertEquals(1, migratedJson.get("rlpv").getAsInt()); + Assert.assertTrue(migratedJson.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.APPEND.name(), migratedJson.get("mt").getAsString()); Assert.assertTrue(migratedJson.has("cs")); Assert.assertTrue(migratedJson.has("eml")); Assert.assertTrue(migratedJson.has("mosn")); @@ -384,7 +392,10 @@ private static Expr predicate(BinaryPredicate.Operator operator, String column, } private static JsonObject imageJson(RoutineLoadJob job) throws IOException { - byte[] image = writeImage(job); + return imageJson(writeImage(job)); + } + + private static JsonObject imageJson(byte[] image) throws IOException { try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(image))) { return JsonParser.parseString(Text.readString(in)).getAsJsonObject(); } From 4394fa3d4d47ccc028e18dade5979a52309a4364 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 07:10:45 +0800 Subject: [PATCH 05/11] [fix](routineload) Complete persistence compatibility coverage ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Routine Load now persists its effective load definition directly, but nested legacy Expr SQL-carrier fields also need stable serialization, legacy image/log fixtures must come from the merge base, and ALTER replay must not apply new leader-side CSV validation to historical journals. Complete those compatibility requirements and add three-FE failover coverage for leader journal write, follower replay, checkpoint, and restart recovery. ### Release note Routine Load jobs preserve effective load clauses across ALTER, follower replay, checkpoint, and FE restart. Legacy images continue to migrate from the original CREATE statement; rollback to an older FE remains structurally readable but does not preserve new ALTER semantics. ### Check List (For Author) - Test: Not run per requested handoff; FE unit and Docker regression coverage were added. - Behavior changed: Yes. Routine Load persistence and ALTER replay retain the current effective definition. - Does this need documentation: Yes. Document rolling-upgrade and rollback limitations. --- .../apache/doris/analysis/MatchPredicate.java | 1 + .../org/apache/doris/analysis/SlotRef.java | 2 + .../apache/doris/analysis/TimeV2Literal.java | 7 + .../load/routineload/RoutineLoadJob.java | 4 +- .../kinesis/KinesisRoutineLoadJob.java | 2 +- .../analysis/ExprGsonSerializationTest.java | 28 +++ .../routineload/KafkaRoutineLoadJobTest.java | 15 ++ .../KinesisRoutineLoadJobTest.java | 17 +- .../RoutineLoadJobPersistenceTest.java | 36 ++- .../AlterRoutineLoadOperationLogTest.java | 36 ++- .../routine-load/a8928245/PROVENANCE.txt | 22 ++ .../a8928245/alter-routine-load-log.b64 | 1 + .../a8928245/routine-load-kafka-image.b64 | 1 + ...ne_load_alter_checkpoint_restart_fe.groovy | 220 ++++++++++++++++++ 14 files changed, 379 insertions(+), 13 deletions(-) create mode 100644 fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt create mode 100644 fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/alter-routine-load-log.b64 create mode 100644 fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/routine-load-kafka-image.b64 create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java index d005827ea061cb..23172942d902f2 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java @@ -72,6 +72,7 @@ public String getName() { private String invertedIndexParserStopwords = ""; private String invertedIndexAnalyzerName = ""; // Fields for SQL generation + @SerializedName("ea") private String explicitAnalyzer = ""; private MatchPredicate() { diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java index a2dee823ad386b..95d3c1b8377f26 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -38,7 +38,9 @@ public class SlotRef extends Expr { @SerializedName("col") private String col; // Used in toSql + @SerializedName("lbl") private String label; + @SerializedName("scp") private List subColPath; // results of analysis diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java index 96a4014bd59a00..8c73083c369fdc 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java @@ -20,14 +20,21 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import com.google.gson.annotations.SerializedName; + public class TimeV2Literal extends LiteralExpr { public static final TimeV2Literal MIN_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, true); public static final TimeV2Literal MAX_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, false); + @SerializedName("h") protected int hour; + @SerializedName("M") protected int minute; + @SerializedName("s") protected int second; + @SerializedName("ms") protected int microsecond; + @SerializedName("neg") protected boolean negative; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index c01edd1ddcd28b..0973a0e1c76059 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2088,6 +2088,7 @@ private void restoreLegacyDefinition() throws UserException { public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; + // Leader-only validation. Replay must accept values written by older FE versions. protected void validateCommonJobProperties(Map jobProperties) throws UserException { validateCsvFormatProperties(jobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { @@ -2106,9 +2107,8 @@ protected void validateCommonJobProperties(Map jobProperties) th } } - // for ALTER ROUTINE LOAD. Validate all common properties before changing any common runtime state. + // Apply ALTER ROUTINE LOAD properties. The leader validates before mutation; replay trusts the journal. protected void modifyCommonJobProperties(Map jobProperties) throws UserException { - validateCommonJobProperties(jobProperties); if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 9c5bac1c7895a2..ea416e48039df4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -684,6 +684,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti throw new DdlException("Only supports modification of PAUSED jobs"); } + validateCommonJobProperties(jobProperties); modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); @@ -698,7 +699,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti private void modifyPropertiesInternal(Map jobProperties, KinesisDataSourceProperties dataSourceProperties) throws UserException { - validateCommonJobProperties(jobProperties); if (dataSourceProperties != null) { List> shardPositions = Lists.newArrayList(); Map customKinesisProperties = Maps.newHashMap(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java index 83b057759ed5a5..4994714b639a2d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java @@ -110,6 +110,26 @@ public void testExprHolderRoundTrip() { Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored)); } + @Test + public void testRoutineLoadExprSqlRoundTrip() throws Exception { + assertExprSqlRoundTrip(new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, + createNamedSlotRef("content"), new StringLiteral("hello"), + Type.BOOLEAN, NullableMode.DEPEND_ON_ARGUMENT, null, false, "english")); + assertExprSqlRoundTrip(new TimeV2Literal(12, 34, 56, 123456, 6, true)); + assertExprSqlRoundTrip(createNamedSlotRef("col1")); + + SlotRef quotedSlot = new SlotRef(null, "a`b"); + quotedSlot.setLabel("`a``b`"); + quotedSlot.setType(Type.BIGINT); + assertExprSqlRoundTrip(quotedSlot); + + SlotRef subPathSlot = createNamedSlotRef("variant_col"); + setDeclaredField(SlotRef.class, subPathSlot, "subColPath", Arrays.asList("nested", "field")); + String subPathJson = GsonUtilsCatalog.GSON.toJson(subPathSlot, Expr.class); + SlotRef restoredSubPath = (SlotRef) GsonUtilsCatalog.GSON.fromJson(subPathJson, Expr.class); + Assertions.assertEquals(Arrays.asList("nested", "field"), restoredSubPath.getSubColPath()); + } + private void assertExprRoundTrip(Class expectedClass, Expr expr) { String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class); Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class); @@ -117,6 +137,14 @@ private void assertExprRoundTrip(Class expectedClass, Expr expr) Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored, Expr.class)); } + private void assertExprSqlRoundTrip(Expr expr) { + String expectedSql = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class); + Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class); + Assertions.assertEquals(expectedSql, + restored.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); + } + private Map, Expr> createExprSamples() throws Exception { LinkedHashMap, Expr> samples = new LinkedHashMap<>(); samples.put(ArithmeticExpr.class, createArithmeticExpr()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index ca0748775e0f28..f8244467502779 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -327,6 +327,21 @@ public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exceptio assertAlterState(imageRoundTrip(follower)); } + @Test + public void testReplayLegacyCsvPropertiesDoesNotRunNewValidation() { + KafkaRoutineLoadJob follower = createPausedJob(); + Map legacyJobProperties = Maps.newHashMap(); + legacyJobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "legacy"); + AlterRoutineLoadJobOperationLog legacyLog = new AlterRoutineLoadJobOperationLog( + follower.getId(), legacyJobProperties, null); + + follower.replayModifyProperties(legacyLog); + + Assert.assertEquals((byte) 'l', follower.getEnclose()); + Map persistedJobProperties = Deencapsulation.getField(follower, "jobProperties"); + Assert.assertEquals("legacy", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + } + private static KafkaRoutineLoadJob createPausedJob() { KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1L, "job1", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index ea64e7e2871b0c..c59309e60f8aa0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -274,10 +274,12 @@ public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exc Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); } - AlterRoutineLoadJobOperationLog log = logCaptor.getValue(); + AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue()); replay.replayModifyProperties(log); - Assert.assertSame(delta, log.getRoutineLoadDesc()); + Assert.assertNotSame(delta, log.getRoutineLoadDesc()); + Assert.assertEquals("\n", log.getRoutineLoadDesc().getLineDelimiter().getSeparator()); + Assert.assertEquals("sequence_col", log.getRoutineLoadDesc().getSequenceColName()); assertAlterResult(leader); assertAlterResult(replay); Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), @@ -428,6 +430,17 @@ private void assertAlterResult(KinesisRoutineLoadJob job) { Assert.assertTrue(job.getEmptyFieldAsNull()); } + private AlterRoutineLoadJobOperationLog journalRoundTrip(AlterRoutineLoadJobOperationLog log) + throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + log.write(out); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return AlterRoutineLoadJobOperationLog.read(in); + } + } + private String checkpointJson(RoutineLoadJob job) throws Exception { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (DataOutputStream out = new DataOutputStream(bytes)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 5c221334e22254..2a8284ad88bec5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -19,15 +19,22 @@ import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeV2Literal; +import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; @@ -82,13 +89,21 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception Separator lineDelimiter = analyzedSeparator("\\n"); List columns = Lists.newArrayList( new ImportColumnDesc("source_col"), - new ImportColumnDesc("mapped_col", new IntLiteral(7L))); - Expr precedingFilter = predicate(BinaryPredicate.Operator.GT, "source_col", 1L); - Expr whereExpr = predicate(BinaryPredicate.Operator.LE, "mapped_col", 10L); + new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true))); + SlotRef matchSlot = namedSlot("content"); + Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, + matchSlot, new StringLiteral("hello world"), Type.BOOLEAN, + NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); + SlotRef quotedSlot = namedSlot("a`b"); + Expr whereExpr = new BinaryPredicate(BinaryPredicate.Operator.GT, quotedSlot, new IntLiteral(10L)); Expr deleteCondition = predicate(BinaryPredicate.Operator.EQ, "delete_flag", 1L); PartitionNamesInfo partitions = new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")); job.setRoutineLoadDesc(new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, precedingFilter, whereExpr, partitions, deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); + String expectedColumnSql = exprToSql(columns.get(1).getExpr()); + String expectedPrecedingSql = exprToSql(precedingFilter); + String expectedWhereSql = exprToSql(whereExpr); + String expectedDeleteSql = exprToSql(deleteCondition); job.desireTaskConcurrentNum = 5; job.maxErrorNum = 17L; @@ -141,6 +156,10 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception Assert.assertNotNull(restored.columnDescs.descs.get(1).getExpr()); Assert.assertNotNull(restored.getPrecedingFilter()); Assert.assertNotNull(restored.getWhereExpr()); + Assert.assertEquals(expectedColumnSql, exprToSql(restored.columnDescs.descs.get(1).getExpr())); + Assert.assertEquals(expectedPrecedingSql, exprToSql(restored.getPrecedingFilter())); + Assert.assertEquals(expectedWhereSql, exprToSql(restored.getWhereExpr())); + Assert.assertEquals(expectedDeleteSql, exprToSql(restored.getDeleteCondition())); Assert.assertEquals("\\x01", restored.getColumnSeparator().getOriSeparator()); Assert.assertEquals("\u0001", restored.getColumnSeparator().getSeparator()); Assert.assertEquals("\\n", restored.getLineDelimiter().getOriSeparator()); @@ -391,6 +410,17 @@ private static Expr predicate(BinaryPredicate.Operator operator, String column, return new BinaryPredicate(operator, new SlotRef(null, column), new IntLiteral(value)); } + private static SlotRef namedSlot(String column) { + SlotRef slotRef = new SlotRef(null, column); + slotRef.setLabel("`" + column.replace("`", "``") + "`"); + slotRef.setType(Type.VARCHAR); + return slotRef; + } + + private static String exprToSql(Expr expr) { + return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + } + private static JsonObject imageJson(RoutineLoadJob job) throws IOException { return imageJson(writeImage(job)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 058cf42f949d54..7fab0f25108f8c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -18,10 +18,18 @@ package org.apache.doris.persist; import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeV2Literal; +import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; @@ -72,16 +80,20 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User Separator lineDelimiter = new Separator("\n", "\\n"); List columns = Lists.newArrayList( new ImportColumnDesc("source_col"), - new ImportColumnDesc("mapped_col", new StringLiteral("mapped_value"))); - BinaryPredicate precedingFilter = new BinaryPredicate(BinaryPredicate.Operator.GT, - new IntLiteral(3L), new IntLiteral(2L)); - BinaryPredicate where = new BinaryPredicate(BinaryPredicate.Operator.EQ, - new StringLiteral("selected"), new StringLiteral("selected")); + new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true))); + Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, + namedSlot("content"), new StringLiteral("hello world"), Type.BOOLEAN, + NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); + Expr where = new BinaryPredicate(BinaryPredicate.Operator.GT, + namedSlot("a`b"), new IntLiteral(10L)); PartitionNamesInfo partitions = new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")); BinaryPredicate deleteCondition = new BinaryPredicate(BinaryPredicate.Operator.EQ, new IntLiteral(1L), new IntLiteral(1L)); RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, precedingFilter, where, partitions, deleteCondition, LoadTask.MergeType.MERGE, "sequence_col"); + String expectedColumnSql = exprToSql(columns.get(1).getExpr()); + String expectedPrecedingSql = exprToSql(precedingFilter); + String expectedWhereSql = exprToSql(where); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, jobProperties, routineLoadDataSourceProperties, routineLoadDesc); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); @@ -115,6 +127,9 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User Assert.assertNotNull(restoredDesc.getColumnsInfo().get(1).getExpr()); Assert.assertNotNull(restoredDesc.getPrecedingFilter()); Assert.assertNotNull(restoredDesc.getFilter()); + Assert.assertEquals(expectedColumnSql, exprToSql(restoredDesc.getColumnsInfo().get(1).getExpr())); + Assert.assertEquals(expectedPrecedingSql, exprToSql(restoredDesc.getPrecedingFilter())); + Assert.assertEquals(expectedWhereSql, exprToSql(restoredDesc.getFilter())); Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp()); Assert.assertEquals(Lists.newArrayList("p1", "p2"), restoredDesc.getPartitionNamesInfo().getPartitionNames()); @@ -146,4 +161,15 @@ private static byte[] loadBase64Fixture(String resource) throws IOException { } } + private static SlotRef namedSlot(String column) { + SlotRef slotRef = new SlotRef(null, column); + slotRef.setLabel("`" + column.replace("`", "``") + "`"); + slotRef.setType(Type.VARCHAR); + return slotRef; + } + + private static String exprToSql(Expr expr) { + return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + } + } diff --git a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt new file mode 100644 index 00000000000000..0d24f78c090320 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt @@ -0,0 +1,22 @@ +These fixtures were generated by the Apache Doris serializer at commit +a8928245ee5712485d43f3dea45872ccb39e9437, the merge base of PR #66634. + +routine-load-kafka-image.b64 contains a PAUSED Kafka routine load job with: + job/database/table IDs: 7001/8001/9001 + broker/topic: 127.0.0.1:9092 / legacy_topic + CREATE load clause: COLUMNS TERMINATED BY '|' + CREATE exec_mem_limit: 33554432 + job properties: max_filter_ratio=0.25, send_batch_parallelism=3, + load_to_single_tablet=true, unique_key_update_mode=UPDATE_FIXED_COLUMNS, + partial_columns=true, partial_update_new_key_behavior=ERROR, enclose='"', + escape='\', empty_field_as_null=true + +Before serialization, the generator changed effective runtime state to separator ',', +sequence column 'alter_sequence', exec_mem_limit 67108864, and +memtable_on_sink_node=true. The a8928245 serializer omitted those runtime-only +changes, documenting why a legacy image can recover only the original CREATE +definition. + +alter-routine-load-log.b64 contains job ID 7001, an empty job-properties map, +and a null datasource-properties object. That serializer predates the +RoutineLoadDesc field in AlterRoutineLoadJobOperationLog. diff --git a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/alter-routine-load-log.b64 b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/alter-routine-load-log.b64 new file mode 100644 index 00000000000000..57d66ed010ffb3 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/alter-routine-load-log.b64 @@ -0,0 +1 @@ +AAAAIXsiam9iSWQiOjcwMDEsImpvYlByb3BlcnRpZXMiOnt9fQ== diff --git a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/routine-load-kafka-image.b64 b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/routine-load-kafka-image.b64 new file mode 100644 index 00000000000000..8322fc582b26f4 --- /dev/null +++ b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/routine-load-kafka-image.b64 @@ -0,0 +1 @@ +AAAEk3siY2xhenoiOiJLYWZrYVJvdXRpbmVMb2FkSm9iIiwiYmwiOiIxMjcuMC4wLjE6OTA5MiIsInRwIjoibGVnYWN5X3RvcGljIiwiY3NrcCI6W10sInByb3AiOnt9LCJpZCI6NzAwMSwibiI6ImxlZ2FjeV9qb2IiLCJkYmlkIjo4MDAxLCJ0YmlkIjo5MDAxLCJkdGNuIjowLCJzdCI6IlBBVVNFRCIsImRzcmMiOiJLQUZLQSIsIm1lbiI6MCwianAiOnsiZW1wdHlfZmllbGRfYXNfbnVsbCI6InRydWUiLCJ1bmlxdWVfa2V5X3VwZGF0ZV9tb2RlIjoiVVBEQVRFX0ZJWEVEX0NPTFVNTlMiLCJwYXJ0aWFsX2NvbHVtbnMiOiJ0cnVlIiwic2VuZF9iYXRjaF9wYXJhbGxlbGlzbSI6IjMiLCJtYXhfZmlsdGVyX3JhdGlvIjoiMC4yNSIsImxvYWRfdG9fc2luZ2xlX3RhYmxldCI6InRydWUiLCJwYXJ0aWFsX3VwZGF0ZV9uZXdfa2V5X2JlaGF2aW9yIjoiRVJST1IiLCJlc2NhcGUiOiJcXCIsImVuY2xvc2UiOiJcIiJ9LCJzdiI6eyJzcWxfbW9kZSI6IjEifSwibWJpcyI6NjAsIm1iciI6MjAwMDAwMDAsIm1ic2IiOjEwNzM3NDE4MjQsInBnIjp7ImNsYXp6IjoiS2Fma2FQcm9ncmVzcyIsInBpdG8iOnt9LCJsZHN0IjoiS0FGS0EifSwiY3RzIjoxNzUzMDAwMDAwMDAwLCJwdHMiOjE3NTMwMDAwMDEwMDAsImV0cyI6LTEsImpzIjp7ImN1cnJlbnRFcnJvclJvd3MiOjAsImN1cnJlbnRUb3RhbFJvd3MiOjAsImVycm9yUm93cyI6MCwidG90YWxSb3dzIjowLCJlcnJvclJvd3NBZnRlclJlc3VtZWQiOjAsInVuc2VsZWN0ZWRSb3dzIjowLCJyZWNlaXZlZEJ5dGVzIjowLCJ0b3RhbFRhc2tFeGN1dGlvblRpbWVNcyI6MSwiY29tbWl0dGVkVGFza051bSI6MCwiYWJvcnRlZFRhc2tOdW0iOjB9LCJvc3RtdCI6eyJvcmlnaW5TdG10IjoiQ1JFQVRFIFJPVVRJTkUgTE9BRCBsZWdhY3lfZGIubGVnYWN5X2pvYiBPTiBzdGFsZV90YWJsZSBDT0xVTU5TIFRFUk1JTkFURUQgQlkgXHUwMDI3fFx1MDAyNyBQUk9QRVJUSUVTIChcImV4ZWNfbWVtX2xpbWl0XCIgXHUwMDNkIFwiMzM1NTQ0MzJcIikgRlJPTSBLQUZLQSAoXCJrYWZrYV9icm9rZXJfbGlzdFwiIFx1MDAzZCBcIjEyNy4wLjAuMTo5MDkyXCIsIFwia2Fma2FfdG9waWNcIiBcdTAwM2QgXCJsZWdhY3lfdG9waWNcIikiLCJpZHgiOjB9LCJ1aSI6eyJ1c2VyIjoiYWRtaW4iLCJob3N0IjoiJSIsImlzRG9tYWluIjpmYWxzZX0sImNtIjoiIn0= diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy new file mode 100644 index 00000000000000..a40a47993608e4 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.Config +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.RoutineLoadTestUtils +import org.awaitility.Awaitility + +import static java.util.concurrent.TimeUnit.SECONDS + +suite("test_routine_load_alter_checkpoint_restart_fe", "docker") { + if (!RoutineLoadTestUtils.isKafkaTestEnabled(context)) { + return + } + + def topicSuffix = System.currentTimeMillis() + def jobName = "test_routine_load_persistence_job_${topicSuffix}" + def topic = "test_routine_load_persistence_topic_${topicSuffix}" + def kafkaBroker = RoutineLoadTestUtils.getKafkaBroker(context) + + def options = new ClusterOptions() + options.setFeNum(3) + options.setBeNum(1) + options.cloudMode = false + options.feConfigs += [ + "edit_log_roll_num=50000" + ] + + def persistedKeys = [ + "columnToColumnExpr", + "column_separator", + "precedingFilter", + "whereExpr", + "exec_mem_limit", + "merge_type" + ] + + docker(options) { + def producer = RoutineLoadTestUtils.createKafkaProducer(kafkaBroker) + Integer stoppedMasterIndex = null + + def runSql = { String query -> sql query } + def readDefinition = { + def showResult = sql "SHOW ROUTINE LOAD FOR ${jobName}" + assertEquals("PAUSED", showResult[0][8].toString()) + def properties = parseJson(showResult[0][11].toString()) + return persistedKeys.collectEntries { key -> + [(key): properties[key].toString()] + } + } + def assertPropertiesEqual = { expected, actual -> + persistedKeys.each { key -> + assertEquals(expected[key], actual[key]) + } + } + def currentJournalId = { + def result = sql """ + SELECT ReplayedJournalId FROM frontends() WHERE IsMaster = 'true' + """ + return result[0][0].toString().toLong() + } + def waitForAllFeReplay = { long targetJournalId -> + Awaitility.await().atMost(90, SECONDS).pollInterval(1, SECONDS).until { + def replayedIds = sql """ + SELECT ReplayedJournalId FROM frontends() WHERE Alive = 'true' + """ + return replayedIds.size() == 3 && replayedIds.every { + it[0].toString().toLong() >= targetJournalId + } + } + } + def latestImageSequence = { frontend -> + def imageDir = new File(frontend.getBasePath(), "doris-meta/image") + long latest = -1L + imageDir.listFiles()?.each { file -> + if (file.name ==~ /^image\.\d+$/) { + latest = Math.max(latest, file.name.substring("image.".length()).toLong()) + } + } + return latest + } + def waitForCheckpoint = { frontend, long targetJournalId -> + Awaitility.await().atMost(150, SECONDS).pollInterval(1, SECONDS).until { + return latestImageSequence(frontend) >= targetJournalId + } + } + def reconnectToCurrentMaster = { + def master = cluster.getMasterFe() + assertNotNull(master) + def jdbcUrl = Config.buildUrlWithDb(master.host, master.queryPort, context.dbName) + context.connectTo(jdbcUrl, context.config.jdbcUser, context.config.jdbcPassword) + } + + sql "DROP TABLE IF EXISTS test_routine_load_alter_checkpoint_restart_tbl" + sql """ + CREATE TABLE test_routine_load_alter_checkpoint_restart_tbl ( + `id` INT NULL, + `source_col` STRING NULL, + `event_date` DATE NULL, + `text1` STRING NULL, + `event_dt` DATETIME NULL, + `text2` STRING NULL, + `mapped_col` BIGINT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + + try { + RoutineLoadTestUtils.sendTestDataToKafka(producer, [topic]) + producer.flush() + + sql """ + CREATE ROUTINE LOAD ${jobName} ON test_routine_load_alter_checkpoint_restart_tbl + COLUMNS TERMINATED BY ",", + COLUMNS(id, source_col, event_date, text1, event_dt, text2, mapped_col = id + 1), + PRECEDING FILTER id > 0, + WHERE mapped_col < 100 + PROPERTIES ( + "max_batch_interval" = "5", + "exec_mem_limit" = "268435456" + ) + FROM KAFKA ( + "kafka_broker_list" = "${kafkaBroker}", + "kafka_topic" = "${topic}", + "property.group.id" = "${jobName}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ) + """ + + RoutineLoadTestUtils.waitForTaskFinish( + runSql, jobName, "test_routine_load_alter_checkpoint_restart_tbl", 0) + sql "PAUSE ROUTINE LOAD FOR ${jobName}" + + def originalDefinition = readDefinition() + def oldMaster = cluster.getMasterFe() + assertNotNull(oldMaster) + stoppedMasterIndex = oldMaster.index + + sql "ALTER ROUTINE LOAD FOR ${jobName} COLUMNS TERMINATED BY '|'" + sql """ + ALTER ROUTINE LOAD FOR ${jobName} + COLUMNS(id, source_col, event_date, text1, event_dt, text2, mapped_col = id + 2) + """ + sql "ALTER ROUTINE LOAD FOR ${jobName} PRECEDING FILTER id > 8" + sql "ALTER ROUTINE LOAD FOR ${jobName} WHERE mapped_col < 50" + + def alteredDefinition = readDefinition() + assertNotEquals(originalDefinition.columnToColumnExpr, alteredDefinition.columnToColumnExpr) + assertEquals("'|'", alteredDefinition.column_separator) + assertNotEquals(originalDefinition.precedingFilter, alteredDefinition.precedingFilter) + assertNotEquals(originalDefinition.whereExpr, alteredDefinition.whereExpr) + + long alterJournalId = currentJournalId() + waitForAllFeReplay(alterJournalId) + assertTrue("ALTER must be tested through journal replay before checkpoint", + latestImageSequence(oldMaster) < alterJournalId) + + cluster.stopFrontends(stoppedMasterIndex) + Awaitility.await().atMost(180, SECONDS).pollInterval(1, SECONDS).until { + def newMaster = cluster.getMasterFe() + return newMaster != null && newMaster.index != stoppedMasterIndex + } + reconnectToCurrentMaster() + + def failoverDefinition = readDefinition() + assertPropertiesEqual(alteredDefinition, failoverDefinition) + + // Force the new leader to roll the next ALTER journal so that checkpoint recovery is also covered. + sql "ADMIN SET FRONTEND CONFIG ('edit_log_roll_num' = '1')" + sql "ALTER ROUTINE LOAD FOR ${jobName} WHERE mapped_col < 40" + def finalDefinition = readDefinition() + assertNotEquals(failoverDefinition.whereExpr, finalDefinition.whereExpr) + + long finalJournalId = currentJournalId() + cluster.startFrontends(stoppedMasterIndex) + stoppedMasterIndex = null + waitForAllFeReplay(finalJournalId) + + def checkpointMaster = cluster.getMasterFe() + waitForCheckpoint(checkpointMaster, finalJournalId) + cluster.restartFrontends() + sleep(30000) + reconnectToCurrentMaster() + + def restartedDefinition = readDefinition() + assertPropertiesEqual(finalDefinition, restartedDefinition) + } finally { + if (stoppedMasterIndex != null) { + try { + cluster.startFrontends(stoppedMasterIndex) + } catch (Exception e) { + logger.warn("Failed to restart old master FE {}: {}", stoppedMasterIndex, e.message) + } + } + try { + reconnectToCurrentMaster() + sql "STOP ROUTINE LOAD FOR ${jobName}" + } catch (Exception e) { + logger.warn("Failed to stop routine load job {}: {}", jobName, e.message) + } + producer.close() + } + } +} From ca46bd976cac9fc7a982ab00d4a3917321ab63cb Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 13:23:56 +0800 Subject: [PATCH 06/11] [refactor](routineload) Persist effective definition in origin SQL ### What problem does this PR solve? Issue Number: N/A Related PR: #64878 Problem Summary: Directly persisting legacy Expr makes Routine Load metadata depend on complete Gson coverage for a broad and evolving expression graph. Keep origStmt as the durable load-definition carrier instead. ALTER logs persist the original ALTER SQL, leader and follower replay merge each delta into a canonical CREATE statement, and image recovery reparses that current effective statement. ### Release note Routine Load ALTER clauses are persisted by updating the stored CREATE statement and replaying ALTER SQL, avoiding a durable dependency on legacy Expr Gson serialization. ### Check List (For Author) - Test: Unit Test - RoutineLoadJobPersistenceTest and KafkaRoutineLoadJobTest: 16 tests passed. - AlterRoutineLoadOperationLogTest and KinesisRoutineLoadJobTest: 13 tests passed in the preceding targeted run. - Behavior changed: Yes. Routine Load load clauses use canonical origin SQL for image and journal recovery. - Does this need documentation: Yes. Document ALTER recovery and rolling-upgrade compatibility. --- .../apache/doris/analysis/MatchPredicate.java | 1 - .../org/apache/doris/analysis/SlotRef.java | 2 - .../apache/doris/analysis/TimeV2Literal.java | 7 - .../apache/doris/load/RoutineLoadDesc.java | 61 ++- .../load/routineload/RoutineLoadJob.java | 125 +++-- .../kafka/KafkaRoutineLoadJob.java | 5 +- .../kinesis/KinesisRoutineLoadJob.java | 5 +- .../commands/AlterRoutineLoadCommand.java | 13 + .../commands/info/CreateRoutineLoadInfo.java | 11 + .../AlterRoutineLoadJobOperationLog.java | 14 +- .../analysis/ExprGsonSerializationTest.java | 28 -- .../routineload/KafkaRoutineLoadJobTest.java | 59 ++- .../KinesisRoutineLoadJobTest.java | 58 ++- .../RoutineLoadJobPersistenceTest.java | 431 +++++------------- .../AlterRoutineLoadOperationLogTest.java | 80 +--- .../routine-load/a8928245/PROVENANCE.txt | 2 +- 16 files changed, 374 insertions(+), 528 deletions(-) diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java index 23172942d902f2..d005827ea061cb 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java @@ -72,7 +72,6 @@ public String getName() { private String invertedIndexParserStopwords = ""; private String invertedIndexAnalyzerName = ""; // Fields for SQL generation - @SerializedName("ea") private String explicitAnalyzer = ""; private MatchPredicate() { diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java index 95d3c1b8377f26..a2dee823ad386b 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -38,9 +38,7 @@ public class SlotRef extends Expr { @SerializedName("col") private String col; // Used in toSql - @SerializedName("lbl") private String label; - @SerializedName("scp") private List subColPath; // results of analysis diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java index 8c73083c369fdc..96a4014bd59a00 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java @@ -20,21 +20,14 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; -import com.google.gson.annotations.SerializedName; - public class TimeV2Literal extends LiteralExpr { public static final TimeV2Literal MIN_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, true); public static final TimeV2Literal MAX_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, false); - @SerializedName("h") protected int hour; - @SerializedName("M") protected int minute; - @SerializedName("s") protected int second; - @SerializedName("ms") protected int microsecond; - @SerializedName("neg") protected boolean negative; /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java index 28a429960d2b71..862b75cbd3ddef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java @@ -18,37 +18,33 @@ package org.apache.doris.load; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.qe.SqlModeHelper; import com.google.common.base.Strings; -import com.google.gson.annotations.SerializedName; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; public class RoutineLoadDesc { - @SerializedName("cs") private final Separator columnSeparator; - @SerializedName("ld") private final Separator lineDelimiter; - @SerializedName("cols") private final List columnsInfo; - @SerializedName("pf") private final Expr precedingFilter; - @SerializedName("f") private final Expr filter; - @SerializedName("dc") private final Expr deleteCondition; - @SerializedName("mt") private LoadTask.MergeType mergeType; // nullable - @SerializedName("pn") private final PartitionNamesInfo partitionNamesInfo; - @SerializedName("sc") private final String sequenceColName; public RoutineLoadDesc(Separator columnSeparator, Separator lineDelimiter, List columnsInfo, @@ -107,6 +103,51 @@ public boolean hasSequenceCol() { return !Strings.isNullOrEmpty(sequenceColName); } + /** + * Convert the effective load clauses to SQL so they can be persisted in RoutineLoadJob.origStmt. + */ + public String toSql() { + List clauses = new ArrayList<>(); + if (columnSeparator != null) { + clauses.add("COLUMNS TERMINATED BY " + SqlUtils.quoteStringLiteral( + columnSeparator.getOriSeparator(), SqlModeHelper.hasNoBackSlashEscapes())); + } + if (columnsInfo != null) { + clauses.add("COLUMNS(" + columnsInfo.stream() + .map(this::columnToSql) + .collect(Collectors.joining(", ")) + ")"); + } + if (precedingFilter != null) { + clauses.add("PRECEDING FILTER " + precedingFilter.accept( + ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); + } + if (filter != null) { + clauses.add("WHERE " + filter.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); + } + if (partitionNamesInfo != null) { + String prefix = partitionNamesInfo.isTemp() ? "TEMPORARY PARTITION(" : "PARTITION("; + clauses.add(prefix + partitionNamesInfo.getPartitionNames().stream() + .map(SqlUtils::getIdentSql) + .collect(Collectors.joining(", ")) + ")"); + } + if (deleteCondition != null) { + clauses.add("DELETE ON " + deleteCondition.accept( + ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); + } + if (hasSequenceCol()) { + clauses.add("ORDER BY " + SqlUtils.getIdentSql(sequenceColName)); + } + return String.join(", ", clauses); + } + + private String columnToSql(ImportColumnDesc columnDesc) { + String sql = SqlUtils.getIdentSql(columnDesc.getColumnName()); + if (columnDesc.getExpr() != null) { + sql += " = " + columnDesc.getExpr().accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + } + return sql; + } + public void analyze() throws UserException { if (mergeType != LoadTask.MergeType.MERGE && deleteCondition != null) { throw new AnalysisException("not support DELETE ON clause when merge type is not MERGE."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 0973a0e1c76059..589bb6331c641d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; +import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -41,6 +42,7 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; +import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; @@ -54,6 +56,7 @@ import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; import org.apache.doris.nereids.load.NereidsStreamLoadPlanner; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.LoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; @@ -181,15 +184,10 @@ public boolean isFinalState() { // this code is used to verify be task request protected long authCode; // protected RoutineLoadDesc routineLoadDesc; // optional - @SerializedName("pni") protected PartitionNamesInfo partitionNamesInfo; // optional - @SerializedName("cds") protected ImportColumnDescs columnDescs; // optional - @SerializedName("pf") protected Expr precedingFilter; // optional - @SerializedName("we") protected Expr whereExpr; // optional - @SerializedName("cs") protected Separator columnSeparator; // optional @SerializedName("lidel") protected Separator lineDelimiter; @@ -236,7 +234,6 @@ public boolean isFinalState() { protected TPartialUpdateNewRowPolicy partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; protected TUniqueKeyUpdateMode uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; - @SerializedName("sc") protected String sequenceCol; @SerializedName("mosn") @@ -266,7 +263,7 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // Keep the original CREATE statement for downgrade compatibility and legacy image migration. + // Persist the current effective load definition as a CREATE statement. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -277,9 +274,7 @@ public boolean isFinalState() { protected String comment = ""; protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); - @SerializedName("mt") - protected LoadTask.MergeType mergeType; - @SerializedName("dc") + protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; protected Expr deleteCondition; // TODO(ml): error sample @@ -1978,20 +1973,9 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } - // Legacy images did not persist mergeType. New images always contain it, including jobs - // without any load clause, so its absence is sufficient to identify the one-time fallback. - boolean isOldImage = mergeType == null; - if (isOldImage) { - mergeType = LoadTask.MergeType.APPEND; - // Legacy images did not persist this create-time session option. Preserve their historical - // post-restart behavior instead of inheriting the image-loading thread's ConnectContext. - memtableOnSinkNode = false; - } try { hydrateJobProperties(); - if (isOldImage) { - restoreLegacyDefinition(); - } + restoreLoadDefinition(); } catch (Exception e) { this.state = JobState.CANCELLED; LOG.warn("error happens when restoring routine load job", e); @@ -2046,40 +2030,85 @@ private void hydrateJobProperties() throws UserException { } } - private void restoreLegacyDefinition() throws UserException { + private void restoreLoadDefinition() throws UserException { + Database database = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); + ConnectContext ctx = createLoadDefinitionContext(database); + try { + ctx.setThreadLocalInfo(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) parsePersistedStatement(origStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + String currentTableName = isMultiTable ? "" : getTableName(); + setRoutineLoadDesc(createRoutineLoadInfo.analyzeLoadProperties( + ctx, database.getName(), currentTableName)); + createRoutineLoadInfo.checkJobProperties(); + execMemLimit = createRoutineLoadInfo.getExecMemLimit(); + } finally { + ctx.cleanup(); + } + } + + protected void replayLoadDefinition(OriginStatement alterStatement) throws UserException { + if (alterStatement == null) { + return; + } + Database database = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); + ConnectContext ctx = createLoadDefinitionContext(database); + try { + ctx.setThreadLocalInfo(); + AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) parsePersistedStatement(alterStatement); + setRoutineLoadDesc(command.analyzeLoadProperties(ctx, this)); + mergeLoadDescToOriginStatement(); + } finally { + ctx.cleanup(); + } + } + + private ConnectContext createLoadDefinitionContext(Database database) { ConnectContext ctx = new ConnectContext(); - ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); + ctx.setDatabase(database.getName()); StatementContext statementContext = new StatementContext(); statementContext.setConnectContext(ctx); ctx.setStatementContext(statementContext); ctx.setEnv(Env.getCurrentEnv()); ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + if (sessionVariables.containsKey(SessionVariable.SQL_MODE)) { + ctx.getSessionVariable().setSqlMode(Long.parseLong(sessionVariables.get(SessionVariable.SQL_MODE))); + } ctx.getState().reset(); - try { - ctx.setThreadLocalInfo(); - NereidsParser nereidsParser = new NereidsParser(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( - origStmt.originStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - // Resolve the current table name by ID so table rename or SWAP TABLE does not leave the - // legacy CREATE statement pointing at a stale table name. - if (!isMultiTable && tableId != 0) { - try { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); - if (db != null) { - db.getTable(tableId).ifPresent( - table -> createRoutineLoadInfo.setTableName(table.getName())); - } - } catch (Exception ignored) { - // Let validate() below surface the original catalog error. - } - } - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); - execMemLimit = createRoutineLoadInfo.getExecMemLimit(); - } finally { - ctx.cleanup(); + return ctx; + } + + protected void mergeLoadDescToOriginStatement() throws UserException { + List columns = + columnDescs == null ? null : Lists.newArrayList(columnDescs.descs); + RoutineLoadDesc loadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol); + StringBuilder sql = new StringBuilder("CREATE ROUTINE LOAD ") + .append(SqlUtils.getIdentSql(name)); + if (!isMultiTable) { + sql.append(" ON ").append(SqlUtils.getIdentSql(getTableName())); + } + sql.append(" WITH ").append(mergeType.name()); + String loadClauseSql = loadDesc.toSql(); + if (!loadClauseSql.isEmpty()) { + sql.append(" ").append(loadClauseSql); } + sql.append(" PROPERTIES (\"exec_mem_limit\" = \"").append(execMemLimit).append("\")"); + sql.append(buildPersistedDataSourceSql()); + origStmt = new OriginStatement(sql.toString(), 0); + } + + private String buildPersistedDataSourceSql() { + if (dataSourceType == LoadDataSourceType.KINESIS) { + return " FROM KINESIS (\"aws.region\" = \"us-east-1\", " + + "\"kinesis_stream\" = \"__routine_load_persistence__\")"; + } + return " FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " + + "\"kafka_topic\" = \"__routine_load_persistence__\")"; + } + + private LogicalPlan parsePersistedStatement(OriginStatement statement) { + return new NereidsParser().parseMultiple(statement.originStmt).get(statement.idx).first; } public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index be124f7c72e6c5..100d4692bcefa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -791,9 +791,10 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); + mergeLoadDescToOriginStatement(); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); + jobProperties, dataSourceProperties, command.getOriginStatement()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -909,7 +910,7 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); - setRoutineLoadDesc(log.getRoutineLoadDesc()); + replayLoadDefinition(log.getOriginStatement()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index ea416e48039df4..146c514e67b6a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -687,9 +687,10 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti validateCommonJobProperties(jobProperties); modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); + mergeLoadDescToOriginStatement(); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); + jobProperties, dataSourceProperties, command.getOriginStatement()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -772,7 +773,7 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); - setRoutineLoadDesc(log.getRoutineLoadDesc()); + replayLoadDefinition(log.getOriginStatement()); } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 367480c5d934d9..3b02c598705bab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; import com.google.common.collect.ImmutableSet; @@ -87,6 +88,7 @@ public class AlterRoutineLoadCommand extends AlterCommand { private final LabelNameInfo labelNameInfo; private final Map loadPropertyMap; private RoutineLoadDesc routineLoadDesc; + private OriginStatement originStatement; private final Map jobProperties; private final Map dataSourceMapProperties; private boolean isPartialUpdate; @@ -149,6 +151,16 @@ public RoutineLoadDesc getRoutineLoadDesc() { return routineLoadDesc; } + public OriginStatement getOriginStatement() { + return originStatement; + } + + /** Analyze only the load-clause delta while replaying the persisted ALTER statement. */ + public RoutineLoadDesc analyzeLoadProperties(ConnectContext ctx, RoutineLoadJob job) throws UserException { + return CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, + job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); + } + @Override public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { validate(ctx); @@ -159,6 +171,7 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { * validate */ public void validate(ConnectContext ctx) throws UserException { + originStatement = ctx.getStatementContext().getOriginStatement(); labelNameInfo.validate(ctx); FeNameFormat.checkCommonName(NAME_TYPE, labelNameInfo.getLabel()); // check routine load job properties include desired concurrent number etc. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java index e8a75c0299b4d6..a24d1a72980c27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java @@ -412,6 +412,17 @@ public void validate(ConnectContext ctx) throws UserException { } } + /** + * Analyze only the load clauses from a persisted CREATE statement. RoutineLoadJob uses this + * during image and journal replay because job and data-source properties have their own + * persisted state. + */ + public RoutineLoadDesc analyzeLoadProperties(ConnectContext ctx, String currentDbName, + String currentTableName) throws UserException { + return checkLoadProperties(ctx, loadPropertyMap, currentDbName, currentTableName, + isMultiTable, mergeType); + } + private void checkDBTable(ConnectContext ctx) throws AnalysisException { labelNameInfo.validate(ctx); dbName = labelNameInfo.getDb(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index 9d8064943c566a..28a77a49e46277 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -19,9 +19,9 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.OriginStatement; import com.google.gson.annotations.SerializedName; @@ -38,8 +38,8 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private Map jobProperties; @SerializedName(value = "dataSourceProperties") private AbstractDataSourceProperties dataSourceProperties; - @SerializedName(value = "routineLoadDesc") - private RoutineLoadDesc routineLoadDesc; + @SerializedName(value = "originStatement") + private OriginStatement originStatement; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { @@ -47,11 +47,11 @@ public AlterRoutineLoadJobOperationLog(long jobId, Map jobProper } public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, - AbstractDataSourceProperties dataSourceProperties, RoutineLoadDesc routineLoadDesc) { + AbstractDataSourceProperties dataSourceProperties, OriginStatement originStatement) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; - this.routineLoadDesc = routineLoadDesc; + this.originStatement = originStatement; } public long getJobId() { @@ -66,8 +66,8 @@ public AbstractDataSourceProperties getDataSourceProperties() { return dataSourceProperties; } - public RoutineLoadDesc getRoutineLoadDesc() { - return routineLoadDesc; + public OriginStatement getOriginStatement() { + return originStatement; } public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java index 4994714b639a2d..83b057759ed5a5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java @@ -110,26 +110,6 @@ public void testExprHolderRoundTrip() { Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored)); } - @Test - public void testRoutineLoadExprSqlRoundTrip() throws Exception { - assertExprSqlRoundTrip(new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, - createNamedSlotRef("content"), new StringLiteral("hello"), - Type.BOOLEAN, NullableMode.DEPEND_ON_ARGUMENT, null, false, "english")); - assertExprSqlRoundTrip(new TimeV2Literal(12, 34, 56, 123456, 6, true)); - assertExprSqlRoundTrip(createNamedSlotRef("col1")); - - SlotRef quotedSlot = new SlotRef(null, "a`b"); - quotedSlot.setLabel("`a``b`"); - quotedSlot.setType(Type.BIGINT); - assertExprSqlRoundTrip(quotedSlot); - - SlotRef subPathSlot = createNamedSlotRef("variant_col"); - setDeclaredField(SlotRef.class, subPathSlot, "subColPath", Arrays.asList("nested", "field")); - String subPathJson = GsonUtilsCatalog.GSON.toJson(subPathSlot, Expr.class); - SlotRef restoredSubPath = (SlotRef) GsonUtilsCatalog.GSON.fromJson(subPathJson, Expr.class); - Assertions.assertEquals(Arrays.asList("nested", "field"), restoredSubPath.getSubColPath()); - } - private void assertExprRoundTrip(Class expectedClass, Expr expr) { String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class); Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class); @@ -137,14 +117,6 @@ private void assertExprRoundTrip(Class expectedClass, Expr expr) Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored, Expr.class)); } - private void assertExprSqlRoundTrip(Expr expr) { - String expectedSql = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class); - Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class); - Assertions.assertEquals(expectedSql, - restored.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); - } - private Map, Expr> createExprSamples() throws Exception { LinkedHashMap, Expr> samples = new LinkedHashMap<>(); samples.put(ArithmeticExpr.class, createArithmeticExpr()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index f8244467502779..321566589770e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -30,6 +30,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; @@ -49,6 +50,7 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRoutineLoadTask; @@ -75,6 +77,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; public class KafkaRoutineLoadJobTest { @@ -102,6 +105,7 @@ public class KafkaRoutineLoadJobTest { @Before public void init() { connectContextStatic = MockedAuth.mockedConnectContext(connectContext, "root", "192.168.1.1"); + Mockito.when(connectContext.getDatabase()).thenReturn("db1"); List partitionNameList = Lists.newArrayList(); partitionNameList.add("p1"); @@ -282,30 +286,53 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro } @Test - public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exception { + public void testAlterPersistsOriginStatementAndCsvPropertiesForReplay() throws Exception { KafkaRoutineLoadJob leader = createPausedJob(); KafkaRoutineLoadJob follower = createPausedJob(); RoutineLoadDesc originalDesc = new RoutineLoadDesc(new Separator("|", "|"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, "original_sequence"); leader.setRoutineLoadDesc(originalDesc); follower.setRoutineLoadDesc(originalDesc); + leader.origStmt = initialOriginStatement(); + follower.origStmt = initialOriginStatement(); Map jobProperties = Maps.newHashMap(); jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); - RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), null, + RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, null); + OriginStatement alterStatement = new OriginStatement( + "ALTER ROUTINE LOAD FOR job1 COLUMNS TERMINATED BY ';'", 0); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(null); Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); + Mockito.when(command.getOriginStatement()).thenReturn(alterStatement); Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(database); + Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("db1"); + Mockito.when(database.getFullName()).thenReturn("db1"); + Mockito.when(database.getTableOrMetaException(1L)).thenReturn(table); + Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("table1"); + Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); AlterRoutineLoadJobOperationLog alterLog; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); Mockito.when(env.getEditLog()).thenReturn(editLog); leader.modifyProperties(command); @@ -314,17 +341,16 @@ public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exceptio ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); alterLog = logCaptor.getValue(); - } - - Assert.assertSame(delta, alterLog.getRoutineLoadDesc()); - Assert.assertEquals(jobProperties, alterLog.getJobProperties()); - assertAlterState(leader); + Assert.assertEquals(alterStatement.originStmt, alterLog.getOriginStatement().originStmt); + Assert.assertEquals(jobProperties, alterLog.getJobProperties()); + assertAlterState(leader); - follower.replayModifyProperties(alterLog); - assertAlterState(follower); + follower.replayModifyProperties(alterLog); + assertAlterState(follower); - assertAlterState(imageRoundTrip(leader)); - assertAlterState(imageRoundTrip(follower)); + assertAlterState(imageRoundTrip(leader)); + assertAlterState(imageRoundTrip(follower)); + } } @Test @@ -350,8 +376,8 @@ private static KafkaRoutineLoadJob createPausedJob() { } private static void assertAlterState(RoutineLoadJob job) { - Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); - Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); + Assert.assertEquals(";", job.getColumnSeparator().getSeparator()); + Assert.assertNull(job.getLineDelimiter()); Assert.assertEquals("original_sequence", job.getSequenceCol()); Assert.assertEquals((byte) '"', job.getEnclose()); Assert.assertEquals((byte) '\\', job.getEscape()); @@ -364,6 +390,13 @@ private static void assertAlterState(RoutineLoadJob job) { Assert.assertEquals("true", persistedJobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } + private static OriginStatement initialOriginStatement() { + return new OriginStatement("CREATE ROUTINE LOAD db1.job1 ON table1 " + + "COLUMNS TERMINATED BY '|', ORDER BY original_sequence " + + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9020\", " + + "\"kafka_topic\" = \"topic1\")", 0); + } + private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) throws Exception { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (DataOutputStream out = new DataOutputStream(bytes)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index c59309e60f8aa0..4ef10196f66d29 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -19,10 +19,14 @@ import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; @@ -35,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.OriginStatement; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -54,6 +59,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -248,7 +254,7 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi } @Test - public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exception { + public void testAlterOriginStatementReplayKeepsCsvCachesInCheckpointParity() throws Exception { KinesisRoutineLoadJob leader = createPausedJobWithInitialLoadDesc(); KinesisRoutineLoadJob replay = createPausedJobWithInitialLoadDesc(); @@ -256,34 +262,53 @@ public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exc jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); - RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), + RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, "sequence_col"); + OriginStatement alterStatement = new OriginStatement( + "ALTER ROUTINE LOAD FOR kinesis_routine_load_job " + + "COLUMNS TERMINATED BY ';', ORDER BY sequence_col", 0); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(null); Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); + Mockito.when(command.getOriginStatement()).thenReturn(alterStatement); Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(database); + Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("db1"); + Mockito.when(database.getFullName()).thenReturn("db1"); + Mockito.when(database.getTableOrMetaException(1L)).thenReturn(table); + Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("table1"); + Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); ArgumentCaptor logCaptor = ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); Mockito.when(env.getEditLog()).thenReturn(editLog); leader.modifyProperties(command); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); + AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue()); + replay.replayModifyProperties(log); + + Assert.assertEquals(alterStatement.originStmt, log.getOriginStatement().originStmt); + assertAlterResult(leader); + assertAlterResult(replay); + Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), + JsonParser.parseString(checkpointJson(replay))); } - - AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue()); - replay.replayModifyProperties(log); - - Assert.assertNotSame(delta, log.getRoutineLoadDesc()); - Assert.assertEquals("\n", log.getRoutineLoadDesc().getLineDelimiter().getSeparator()); - Assert.assertEquals("sequence_col", log.getRoutineLoadDesc().getSequenceColName()); - assertAlterResult(leader); - assertAlterResult(replay); - Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), - JsonParser.parseString(checkpointJson(replay))); } @Test @@ -418,12 +443,15 @@ private KinesisRoutineLoadJob createPausedJobWithInitialLoadDesc() { Deencapsulation.setField(job, "createTimestamp", 123L); job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator("|", "|"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, null)); + job.origStmt = new OriginStatement("CREATE ROUTINE LOAD db1.kinesis_routine_load_job ON table1 " + + "COLUMNS TERMINATED BY '|' FROM KINESIS " + + "(\"aws.region\" = \"us-east-1\", \"kinesis_stream\" = \"stream-1\")", 0); return job; } private void assertAlterResult(KinesisRoutineLoadJob job) { - Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); - Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); + Assert.assertEquals(";", job.getColumnSeparator().getSeparator()); + Assert.assertNull(job.getLineDelimiter()); Assert.assertEquals("sequence_col", job.getSequenceCol()); Assert.assertEquals((byte) '"', job.getEnclose()); Assert.assertEquals((byte) '\\', job.getEscape()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 2a8284ad88bec5..b315c93ddd583a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -17,44 +17,28 @@ package org.apache.doris.load.routineload; +import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.BinaryPredicate; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.StringLiteral; -import org.apache.doris.analysis.TimeV2Literal; -import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; -import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; -import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; -import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; -import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; -import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; -import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.qe.OriginStatement; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; -import org.apache.doris.thrift.TUniqueKeyUpdateMode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.junit.Assert; @@ -71,7 +55,6 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.List; -import java.util.Map; import java.util.Optional; public class RoutineLoadJobPersistenceTest { @@ -79,325 +62,128 @@ public class RoutineLoadJobPersistenceTest { "/upgrade/routine-load/a8928245/routine-load-kafka-image.b64"; @Test - public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "direct_job", 1002L, - 1003L, "127.0.0.1:9092", "direct_topic", UserIdentity.ADMIN); + public void testImageRestoresLoadDefinitionFromOrigStmt() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "image_job", 8001L, + 9001L, "127.0.0.1:9092", "image_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("this is deliberately not valid SQL", 0); - - Separator columnSeparator = analyzedSeparator("\\x01"); - Separator lineDelimiter = analyzedSeparator("\\n"); - List columns = Lists.newArrayList( - new ImportColumnDesc("source_col"), - new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true))); - SlotRef matchSlot = namedSlot("content"); - Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, - matchSlot, new StringLiteral("hello world"), Type.BOOLEAN, - NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); - SlotRef quotedSlot = namedSlot("a`b"); - Expr whereExpr = new BinaryPredicate(BinaryPredicate.Operator.GT, quotedSlot, new IntLiteral(10L)); - Expr deleteCondition = predicate(BinaryPredicate.Operator.EQ, "delete_flag", 1L); - PartitionNamesInfo partitions = new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")); - job.setRoutineLoadDesc(new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, - precedingFilter, whereExpr, partitions, deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); - String expectedColumnSql = exprToSql(columns.get(1).getExpr()); - String expectedPrecedingSql = exprToSql(precedingFilter); - String expectedWhereSql = exprToSql(whereExpr); - String expectedDeleteSql = exprToSql(deleteCondition); - - job.desireTaskConcurrentNum = 5; - job.maxErrorNum = 17L; - job.maxBatchIntervalS = 23L; - job.maxBatchRows = 300001L; - job.maxBatchSizeBytes = 104857601L; - job.execMemLimit = 345678901L; - job.maxFilterRatio = 0.99; - job.sendBatchParallelism = 99; - job.loadToSingleTablet = false; + job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.image_job ON stale_table " + + "COLUMNS TERMINATED BY '|', " + + "COLUMNS(source_col, mapped_col = source_col + 1), " + + "PRECEDING FILTER source_col > 1, WHERE mapped_col <= 10 " + + "PROPERTIES (\"exec_mem_limit\" = \"345678901\") " + + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " + + "\"kafka_topic\" = \"image_topic\")", 0); + + job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator(",", ","), analyzedSeparator("\\n"), + Lists.newArrayList(new ImportColumnDesc("wrong_column")), + null, null, null, null, LoadTask.MergeType.APPEND, null)); job.memtableOnSinkNode = true; - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, "0.25"); - jobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, "4"); - jobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, "true"); - jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FIXED_COLUMNS"); - jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); - jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, "ERROR"); - jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); - jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); - jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); - job.jobProperties = jobProperties; - JsonObject json = imageJson(job); Assert.assertTrue(json.has("ostmt")); - Assert.assertEquals(LoadTask.MergeType.MERGE.name(), json.get("mt").getAsString()); - for (String key : Lists.newArrayList( - "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc", "eml", "mosn")) { - Assert.assertTrue("missing direct-state key " + key, json.has(key)); + Assert.assertTrue(json.has("eml")); + Assert.assertTrue(json.has("mosn")); + Assert.assertTrue(json.has("lidel")); + for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "sc", "mt", "dc")) { + Assert.assertFalse("load definition must only be persisted through origStmt: " + key, json.has(key)); } - Assert.assertFalse(json.has("ld")); - Assert.assertEquals("\\x01", json.getAsJsonObject("cs").get("os").getAsString()); - Assert.assertEquals("\u0001", json.getAsJsonObject("cs").get("s").getAsString()); - Assert.assertEquals("\\n", json.getAsJsonObject("lidel").get("os").getAsString()); - Assert.assertEquals("\n", json.getAsJsonObject("lidel").get("s").getAsString()); - Assert.assertEquals(2, json.getAsJsonObject("cds").getAsJsonArray("des").size()); RoutineLoadJob restored; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + try (MockedStatic ignored = mockCatalog()) { restored = imageRoundTrip(job); - envStatic.verifyNoInteractions(); } - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals(Lists.newArrayList("p1", "p2"), restored.getPartitionNamesInfo().getPartitionNames()); - Assert.assertEquals(2, restored.columnDescs.descs.size()); - Assert.assertEquals("source_col", restored.columnDescs.descs.get(0).getColumnName()); - Assert.assertEquals("mapped_col", restored.columnDescs.descs.get(1).getColumnName()); - Assert.assertNotNull(restored.columnDescs.descs.get(1).getExpr()); + Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals(2, restored.getColumnExprDescs().descs.size()); + Assert.assertEquals("source_col", restored.getColumnExprDescs().descs.get(0).getColumnName()); + Assert.assertEquals("mapped_col", restored.getColumnExprDescs().descs.get(1).getColumnName()); Assert.assertNotNull(restored.getPrecedingFilter()); Assert.assertNotNull(restored.getWhereExpr()); - Assert.assertEquals(expectedColumnSql, exprToSql(restored.columnDescs.descs.get(1).getExpr())); - Assert.assertEquals(expectedPrecedingSql, exprToSql(restored.getPrecedingFilter())); - Assert.assertEquals(expectedWhereSql, exprToSql(restored.getWhereExpr())); - Assert.assertEquals(expectedDeleteSql, exprToSql(restored.getDeleteCondition())); - Assert.assertEquals("\\x01", restored.getColumnSeparator().getOriSeparator()); - Assert.assertEquals("\u0001", restored.getColumnSeparator().getSeparator()); - Assert.assertEquals("\\n", restored.getLineDelimiter().getOriSeparator()); - Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); - Assert.assertEquals("seq_col", restored.getSequenceCol()); - Assert.assertEquals(LoadTask.MergeType.MERGE, restored.getMergeType()); - Assert.assertNotNull(restored.getDeleteCondition()); Assert.assertEquals(345678901L, restored.getMemLimit()); + Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); Assert.assertTrue(restored.isMemtableOnSinkNode()); - Assert.assertEquals(5, restored.desireTaskConcurrentNum); - Assert.assertEquals(17L, restored.maxErrorNum); - Assert.assertEquals(23L, restored.getMaxBatchIntervalS()); - Assert.assertEquals(300001L, restored.getMaxBatchRows()); - Assert.assertEquals(104857601L, restored.getMaxBatchSizeBytes()); - - NereidsRoutineLoadTaskInfo taskInfo = restored.toNereidsRoutineLoadTaskInfo(); - Assert.assertEquals(345678901L, taskInfo.getMemLimit()); - Assert.assertEquals(0.25, taskInfo.getMaxFilterRatio(), 0.0); - Assert.assertEquals(4, taskInfo.getSendBatchParallelism()); - Assert.assertTrue(taskInfo.isLoadToSingleTablet()); - Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, taskInfo.getUniqueKeyUpdateMode()); - Assert.assertTrue(taskInfo.isFixedPartialUpdate()); - Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, taskInfo.getPartialUpdateNewRowPolicy()); - Assert.assertEquals((byte) '"', taskInfo.getEnclose()); - Assert.assertEquals((byte) '\\', taskInfo.getEscape()); - Assert.assertTrue(taskInfo.getEmptyFieldAsNull()); - Assert.assertTrue(taskInfo.isMemtableOnSinkNode()); - Assert.assertEquals(LoadTask.MergeType.MERGE, taskInfo.getMergeType()); - Assert.assertNotNull(taskInfo.getDeleteCondition()); - Assert.assertEquals("seq_col", taskInfo.getSequenceCol()); - Assert.assertEquals(Lists.newArrayList("p1", "p2"), - taskInfo.getPartitionNamesInfo().getPartitionNames()); - Assert.assertEquals(2, taskInfo.getColumnExprDescs().descs.size()); - Assert.assertNotNull(taskInfo.getPrecedingFilter()); - Assert.assertNotNull(taskInfo.getWhereExpr()); - Assert.assertEquals("\u0001", taskInfo.getColumnSeparator().getSeparator()); - Assert.assertEquals("\n", taskInfo.getLineDelimiter().getSeparator()); } @Test - public void testDirectStateImageWithNoLoadClausesDoesNotFallback() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "empty_job", 2002L, - 2003L, "127.0.0.1:9092", "empty_topic", UserIdentity.ADMIN); + public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "alter_job", 8001L, + 9001L, "127.0.0.1:9092", "alter_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("also not valid SQL", 0); - - JsonObject json = imageJson(job); - Assert.assertTrue(json.has("ostmt")); - Assert.assertEquals(LoadTask.MergeType.APPEND.name(), json.get("mt").getAsString()); - for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "lidel", "sc", "dc")) { - Assert.assertFalse("unexpected nullable direct-state key " + key, json.has(key)); + job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.alter_job ON current_table " + + "COLUMNS TERMINATED BY ',', " + + "COLUMNS(source_col, mapped_col = source_col + 1), " + + "PRECEDING FILTER source_col > 1, WHERE mapped_col < 100, ORDER BY seq_col " + + "PROPERTIES (\"exec_mem_limit\" = \"268435456\") " + + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " + + "\"kafka_topic\" = \"alter_topic\")", 0); + job.execMemLimit = 268435456L; + job.setRoutineLoadDesc(initialLoadDesc()); + + try (MockedStatic ignored = mockCatalog()) { + job.replayLoadDefinition(new OriginStatement( + "ALTER ROUTINE LOAD FOR alter_job COLUMNS TERMINATED BY '|', WHERE mapped_col < 50", 0)); + job.replayLoadDefinition(new OriginStatement( + "ALTER ROUTINE LOAD FOR alter_job " + + "PRECEDING FILTER content MATCH_ANY 'hello' USING ANALYZER 'english'", 0)); } + Assert.assertTrue(job.origStmt.originStmt.startsWith("CREATE ROUTINE LOAD")); + Assert.assertTrue(job.origStmt.originStmt.contains("COLUMNS TERMINATED BY \"|\"")); + Assert.assertTrue(job.origStmt.originStmt.contains("COLUMNS(")); + Assert.assertTrue(job.origStmt.originStmt.contains("WHERE")); + Assert.assertTrue(job.origStmt.originStmt.contains("PRECEDING FILTER")); + Assert.assertTrue(job.origStmt.originStmt.contains("USING ANALYZER")); + Assert.assertTrue(job.origStmt.originStmt.contains("ORDER BY `seq_col`")); + + JsonObject expectedProperties = JsonParser.parseString(job.jobPropertiesToJsonString()).getAsJsonObject(); RoutineLoadJob restored; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + try (MockedStatic ignored = mockCatalog()) { restored = imageRoundTrip(job); - envStatic.verifyNoInteractions(); } - - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertNull(restored.getPartitionNamesInfo()); - Assert.assertNull(restored.columnDescs); - Assert.assertNull(restored.getPrecedingFilter()); - Assert.assertNull(restored.getWhereExpr()); - Assert.assertNull(restored.getColumnSeparator()); - Assert.assertNull(restored.getLineDelimiter()); - Assert.assertNull(restored.getSequenceCol()); - Assert.assertNull(restored.getDeleteCondition()); - Assert.assertEquals(LoadTask.MergeType.APPEND, restored.getMergeType()); + JsonObject restoredProperties = JsonParser.parseString(restored.jobPropertiesToJsonString()).getAsJsonObject(); + for (String key : Lists.newArrayList("column_separator", "precedingFilter", + "whereExpr", "sequence_col", "merge_type", "exec_mem_limit")) { + Assert.assertEquals(key, expectedProperties.get(key), restoredProperties.get(key)); + } + Assert.assertTrue(restoredProperties.get("columnToColumnExpr").getAsString().contains("mapped_col=")); + Assert.assertEquals(job.origStmt.originStmt, restored.origStmt.originStmt); } @Test - public void testLegacyImageMigratesOnce() throws Exception { - Env env = Mockito.mock(Env.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(8001L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("legacy_db")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrAnalysisException("legacy_db")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("legacy_db"); - Mockito.when(database.getTable(9001L)).thenReturn(Optional.of((Table) table)); - Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("current_table"); - Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(table.getEnableUniqueKeyMergeOnWrite()).thenReturn(true); - + public void testLegacyImageContinuesToRestoreFromOrigStmt() throws Exception { byte[] legacyImage = loadBase64Fixture(LEGACY_IMAGE); JsonObject legacyJson = imageJson(legacyImage); - Assert.assertFalse(legacyJson.has("mt")); Assert.assertTrue(legacyJson.has("ostmt")); + Assert.assertFalse(legacyJson.has("mt")); - RoutineLoadJob migrated; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - migrated = readImage(legacyImage); - } - - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, migrated.getState()); - Assert.assertEquals("|", migrated.getColumnSeparator().getOriSeparator()); - Assert.assertEquals("|", migrated.getColumnSeparator().getSeparator()); - Assert.assertNull(migrated.getSequenceCol()); - Assert.assertEquals(33554432L, migrated.getMemLimit()); - Assert.assertEquals(0.25, migrated.getMaxFilterRatio(), 0.0); - Assert.assertEquals(3, migrated.getSendBatchParallelism()); - Assert.assertTrue(migrated.isLoadToSingleTablet()); - Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, migrated.getUniqueKeyUpdateMode()); - Assert.assertTrue(migrated.isFixedPartialUpdate()); - Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, migrated.partialUpdateNewKeyPolicy); - Assert.assertEquals((byte) '"', migrated.getEnclose()); - Assert.assertEquals((byte) '\\', migrated.getEscape()); - Assert.assertTrue(migrated.getEmptyFieldAsNull()); - Assert.assertFalse(migrated.isMemtableOnSinkNode()); - - JsonObject migratedJson = imageJson(migrated); - Assert.assertTrue(migratedJson.has("ostmt")); - Assert.assertEquals(LoadTask.MergeType.APPEND.name(), migratedJson.get("mt").getAsString()); - Assert.assertTrue(migratedJson.has("cs")); - Assert.assertTrue(migratedJson.has("eml")); - Assert.assertTrue(migratedJson.has("mosn")); - migrated.origStmt = new OriginStatement("invalid after successful migration", 0); - - RoutineLoadJob restoredAgain; - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - restoredAgain = imageRoundTrip(migrated); - envStatic.verifyNoInteractions(); + RoutineLoadJob restored; + try (MockedStatic ignored = mockCatalog()) { + restored = readImage(legacyImage); } - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restoredAgain.getState()); - Assert.assertEquals("|", restoredAgain.getColumnSeparator().getSeparator()); - Assert.assertEquals(33554432L, restoredAgain.getMemLimit()); - Assert.assertFalse(restoredAgain.isMemtableOnSinkNode()); - } - - @Test - public void testKafkaDerivedStateIsRebuiltFromDurableProperties() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(3001L, "kafka_derived", 3002L, - 3003L, "127.0.0.1:9092", "derived_topic", UserIdentity.ADMIN); - job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); - Map customProperties = Maps.newHashMap(); - customProperties.put("client.id", "durable-client"); - customProperties.put(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName(), "OFFSET_BEGINNING"); - Deencapsulation.setField(job, "customProperties", customProperties); - Deencapsulation.setField(job, "customKafkaPartitions", Lists.newArrayList(9)); - Deencapsulation.setField(job, "currentKafkaPartitions", Lists.newArrayList(1, 2)); - Deencapsulation.setField(job, "convertedCustomProperties", - Maps.newHashMap(ImmutableMap.of("stale", "value"))); - Deencapsulation.setField(job, "cachedPartitionWithLatestOffsets", - Maps.newHashMap(ImmutableMap.of(1, 100L))); - Deencapsulation.setField(job, "newCurrentKafkaPartition", Lists.newArrayList(3)); - Deencapsulation.setField(job, "kafkaDefaultOffSet", "OFFSET_END"); - - JsonObject json = imageJson(job); - Assert.assertEquals("127.0.0.1:9092", json.get("bl").getAsString()); - Assert.assertEquals("derived_topic", json.get("tp").getAsString()); - Assert.assertEquals("durable-client", json.getAsJsonObject("prop").get("client.id").getAsString()); - Assert.assertEquals(1, json.getAsJsonArray("cskp").size()); - assertNoJavaFieldNames(json, "currentKafkaPartitions", "convertedCustomProperties", - "cachedPartitionWithLatestOffsets", "newCurrentKafkaPartition", "kafkaDefaultOffSet"); - - KafkaRoutineLoadJob restored = (KafkaRoutineLoadJob) imageRoundTrip(job); - Assert.assertEquals("127.0.0.1:9092", restored.getBrokerList()); - Assert.assertEquals("derived_topic", restored.getTopic()); - Assert.assertEquals(Lists.newArrayList(9), Deencapsulation.getField(restored, "customKafkaPartitions")); - Assert.assertTrue(((List) Deencapsulation.getField(restored, "currentKafkaPartitions")).isEmpty()); - Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); - Assert.assertTrue(((Map) Deencapsulation.getField( - restored, "cachedPartitionWithLatestOffsets")).isEmpty()); - Assert.assertEquals("", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); - Env env = Mockito.mock(Env.class); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - restored.prepare(); - } - Assert.assertEquals("durable-client", restored.getConvertedCustomProperties().get("client.id")); - Assert.assertFalse(restored.getConvertedCustomProperties().containsKey( - KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); - Assert.assertEquals("OFFSET_BEGINNING", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals(33554432L, restored.getMemLimit()); + Assert.assertFalse(restored.isMemtableOnSinkNode()); + + JsonObject newImage = imageJson(restored); + Assert.assertTrue(newImage.has("ostmt")); + Assert.assertFalse(newImage.has("mt")); + Assert.assertFalse(newImage.has("cs")); } - @Test - public void testKinesisDerivedStateIsRebuiltFromDurableProperties() throws Exception { - KinesisRoutineLoadJob job = new KinesisRoutineLoadJob(4001L, "kinesis_derived", 4002L, - 4003L, "us-east-1", "derived_stream", UserIdentity.ADMIN); - job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); - Deencapsulation.setField(job, "endpoint", "https://kinesis.example.test"); - Map customProperties = Maps.newHashMap(); - customProperties.put("client.setting", "durable-value"); - customProperties.put("kinesis_default_pos", "TRIM_HORIZON"); - Deencapsulation.setField(job, "customProperties", customProperties); - Deencapsulation.setField(job, "customKinesisShards", Lists.newArrayList("custom-shard")); - Deencapsulation.setField(job, "openKinesisShards", Lists.newArrayList("open-shard")); - Deencapsulation.setField(job, "closedKinesisShards", Lists.newArrayList("closed-shard")); - Deencapsulation.setField(job, "convertedCustomProperties", - Maps.newHashMap(ImmutableMap.of("stale", "value"))); - Deencapsulation.setField(job, "cachedShardWithMillsBehindLatest", - Maps.newHashMap(ImmutableMap.of("open-shard", 99L))); - Deencapsulation.setField(job, "newCurrentKinesisShards", Lists.newArrayList("new-shard")); - Deencapsulation.setField(job, "kinesisDefaultPosition", "LATEST"); - - JsonObject json = imageJson(job); - Assert.assertEquals("us-east-1", json.get("rg").getAsString()); - Assert.assertEquals("derived_stream", json.get("stm").getAsString()); - Assert.assertEquals("https://kinesis.example.test", json.get("ep").getAsString()); - Assert.assertEquals("durable-value", - json.getAsJsonObject("prop").get("client.setting").getAsString()); - Assert.assertEquals("custom-shard", json.getAsJsonArray("csks").get(0).getAsString()); - Assert.assertEquals("open-shard", json.getAsJsonArray("opks").get(0).getAsString()); - Assert.assertEquals("closed-shard", json.getAsJsonArray("clks").get(0).getAsString()); - assertNoJavaFieldNames(json, "convertedCustomProperties", "cachedShardWithMillsBehindLatest", - "newCurrentKinesisShards", "kinesisDefaultPosition"); - - KinesisRoutineLoadJob restored = (KinesisRoutineLoadJob) imageRoundTrip(job); - Assert.assertEquals("us-east-1", restored.getRegion()); - Assert.assertEquals("derived_stream", restored.getStream()); - Assert.assertEquals("https://kinesis.example.test", restored.getEndpoint()); - Assert.assertEquals(Lists.newArrayList("custom-shard"), - Deencapsulation.getField(restored, "customKinesisShards")); - Assert.assertEquals(Lists.newArrayList("open-shard"), - Deencapsulation.getField(restored, "openKinesisShards")); - Assert.assertEquals(Lists.newArrayList("closed-shard"), - Deencapsulation.getField(restored, "closedKinesisShards")); - Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); - Assert.assertTrue(((Map) Deencapsulation.getField( - restored, "cachedShardWithMillsBehindLatest")).isEmpty()); - Assert.assertTrue(((List) Deencapsulation.getField(restored, "newCurrentKinesisShards")).isEmpty()); - Assert.assertEquals("", Deencapsulation.getField(restored, "kinesisDefaultPosition")); - - restored.prepare(); - Assert.assertEquals("durable-value", restored.getConvertedCustomProperties().get("client.setting")); - Assert.assertEquals("TRIM_HORIZON", - restored.getConvertedCustomProperties().get("kinesis_default_pos")); - Assert.assertEquals("TRIM_HORIZON", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + private static RoutineLoadDesc initialLoadDesc() { + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new ArithmeticExpr(ArithmeticExpr.Operator.ADD, + new SlotRef(null, "source_col"), new IntLiteral(1L), + Type.BIGINT, NullableMode.ALWAYS_NOT_NULLABLE, false))); + Expr preceding = new BinaryPredicate(BinaryPredicate.Operator.GT, + new SlotRef(null, "source_col"), new IntLiteral(1L)); + Expr where = new BinaryPredicate(BinaryPredicate.Operator.LT, + new SlotRef(null, "mapped_col"), new IntLiteral(100L)); + return new RoutineLoadDesc(new Separator(",", ","), null, columns, + preceding, where, null, null, LoadTask.MergeType.APPEND, "seq_col"); } private static Separator analyzedSeparator(String value) throws Exception { @@ -406,19 +192,30 @@ private static Separator analyzedSeparator(String value) throws Exception { return separator; } - private static Expr predicate(BinaryPredicate.Operator operator, String column, long value) { - return new BinaryPredicate(operator, new SlotRef(null, column), new IntLiteral(value)); - } - - private static SlotRef namedSlot(String column) { - SlotRef slotRef = new SlotRef(null, column); - slotRef.setLabel("`" + column.replace("`", "``") + "`"); - slotRef.setType(Type.VARCHAR); - return slotRef; - } + private static MockedStatic mockCatalog() throws Exception { + Env env = Mockito.mock(Env.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); + Mockito.when(catalog.getDb(8001L)).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDb("legacy_db")).thenReturn(Optional.of(database)); + Mockito.when(catalog.getDbOrMetaException(8001L)).thenReturn(database); + Mockito.when(catalog.getDbOrAnalysisException("legacy_db")).thenReturn(database); + Mockito.when(database.getName()).thenReturn("legacy_db"); + Mockito.when(database.getFullName()).thenReturn("legacy_db"); + Mockito.when(database.getTableOrMetaException(9001L)).thenReturn(table); + Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); + Mockito.when(table.getName()).thenReturn("current_table"); + Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); - private static String exprToSql(Expr expr) { - return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + MockedStatic envStatic = Mockito.mockStatic(Env.class); + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + return envStatic; } private static JsonObject imageJson(RoutineLoadJob job) throws IOException { @@ -458,10 +255,4 @@ private static byte[] loadBase64Fixture(String resource) throws IOException { return Base64.getDecoder().decode(base64); } } - - private static void assertNoJavaFieldNames(JsonObject json, String... fieldNames) { - for (String fieldName : fieldNames) { - Assert.assertFalse("derived field leaked into image: " + fieldName, json.has(fieldName)); - } - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 7fab0f25108f8c..afecaedc57eaa6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -17,30 +17,13 @@ package org.apache.doris.persist; -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; -import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.MatchPredicate; -import org.apache.doris.analysis.Separator; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.StringLiteral; -import org.apache.doris.analysis.TimeV2Literal; -import org.apache.doris.analysis.ToSqlParams; -import org.apache.doris.catalog.Function.NullableMode; -import org.apache.doris.catalog.Type; -import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.load.RoutineLoadDesc; -import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.OriginStatement; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -53,7 +36,6 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Base64; -import java.util.List; import java.util.Map; public class AlterRoutineLoadOperationLogTest { @@ -76,26 +58,10 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); - Separator columnSeparator = new Separator(",", "\\x2c"); - Separator lineDelimiter = new Separator("\n", "\\n"); - List columns = Lists.newArrayList( - new ImportColumnDesc("source_col"), - new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true))); - Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, - namedSlot("content"), new StringLiteral("hello world"), Type.BOOLEAN, - NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); - Expr where = new BinaryPredicate(BinaryPredicate.Operator.GT, - namedSlot("a`b"), new IntLiteral(10L)); - PartitionNamesInfo partitions = new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")); - BinaryPredicate deleteCondition = new BinaryPredicate(BinaryPredicate.Operator.EQ, - new IntLiteral(1L), new IntLiteral(1L)); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, - precedingFilter, where, partitions, deleteCondition, LoadTask.MergeType.MERGE, "sequence_col"); - String expectedColumnSql = exprToSql(columns.get(1).getExpr()); - String expectedPrecedingSql = exprToSql(precedingFilter); - String expectedWhereSql = exprToSql(where); + OriginStatement originStatement = new OriginStatement( + "ALTER ROUTINE LOAD FOR job WHERE mapped_col > 10", 0); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties, routineLoadDesc); + jobProperties, routineLoadDataSourceProperties, originStatement); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (DataOutputStream out = new DataOutputStream(bytes)) { log.write(out); @@ -116,38 +82,19 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); - RoutineLoadDesc restoredDesc = log2.getRoutineLoadDesc(); - Assert.assertEquals(",", restoredDesc.getColumnSeparator().getSeparator()); - Assert.assertEquals("\\x2c", restoredDesc.getColumnSeparator().getOriSeparator()); - Assert.assertEquals("\n", restoredDesc.getLineDelimiter().getSeparator()); - Assert.assertEquals("\\n", restoredDesc.getLineDelimiter().getOriSeparator()); - Assert.assertEquals(2, restoredDesc.getColumnsInfo().size()); - Assert.assertEquals("source_col", restoredDesc.getColumnsInfo().get(0).getColumnName()); - Assert.assertEquals("mapped_col", restoredDesc.getColumnsInfo().get(1).getColumnName()); - Assert.assertNotNull(restoredDesc.getColumnsInfo().get(1).getExpr()); - Assert.assertNotNull(restoredDesc.getPrecedingFilter()); - Assert.assertNotNull(restoredDesc.getFilter()); - Assert.assertEquals(expectedColumnSql, exprToSql(restoredDesc.getColumnsInfo().get(1).getExpr())); - Assert.assertEquals(expectedPrecedingSql, exprToSql(restoredDesc.getPrecedingFilter())); - Assert.assertEquals(expectedWhereSql, exprToSql(restoredDesc.getFilter())); - Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp()); - Assert.assertEquals(Lists.newArrayList("p1", "p2"), - restoredDesc.getPartitionNamesInfo().getPartitionNames()); - Assert.assertNotNull(restoredDesc.getDeleteCondition()); - Assert.assertEquals(LoadTask.MergeType.MERGE, restoredDesc.getMergeType()); - Assert.assertEquals("sequence_col", restoredDesc.getSequenceColName()); - Assert.assertEquals(GsonUtils.GSON.toJson(routineLoadDesc), GsonUtils.GSON.toJson(restoredDesc)); + Assert.assertEquals(originStatement.originStmt, log2.getOriginStatement().originStmt); + Assert.assertEquals(originStatement.idx, log2.getOriginStatement().idx); } @Test - public void testDeserializeLegacyLogWithoutRoutineLoadDesc() throws IOException { + public void testDeserializeLegacyLogWithoutOriginStatement() throws IOException { byte[] bytes = loadBase64Fixture(A8928245_LEGACY_LOG); try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes))) { AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); Assert.assertEquals(7001L, log.getJobId()); Assert.assertTrue(log.getJobProperties().isEmpty()); Assert.assertNull(log.getDataSourceProperties()); - Assert.assertNull(log.getRoutineLoadDesc()); + Assert.assertNull(log.getOriginStatement()); } } @@ -161,15 +108,4 @@ private static byte[] loadBase64Fixture(String resource) throws IOException { } } - private static SlotRef namedSlot(String column) { - SlotRef slotRef = new SlotRef(null, column); - slotRef.setLabel("`" + column.replace("`", "``") + "`"); - slotRef.setType(Type.VARCHAR); - return slotRef; - } - - private static String exprToSql(Expr expr) { - return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - } - } diff --git a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt index 0d24f78c090320..156e15b3ed20a3 100644 --- a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt +++ b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt @@ -19,4 +19,4 @@ definition. alter-routine-load-log.b64 contains job ID 7001, an empty job-properties map, and a null datasource-properties object. That serializer predates the -RoutineLoadDesc field in AlterRoutineLoadJobOperationLog. +originStatement field in AlterRoutineLoadJobOperationLog. From d155dceec9a8fc14b4068597ae348153d18a92aa Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 13:44:16 +0800 Subject: [PATCH 07/11] [refactor](routineload) Make origin SQL authoritative ### What problem does this PR solve? Issue Number: close #66633 Related PR: #66634 Problem Summary: Routine Load cannot safely use the legacy Expr object graph as an image or journal compatibility surface. Keep the current effective load definition in origStmt, remove the duplicate execMemLimit JSON source, and cover all SQL-representable load clauses through CREATE image restore, ALTER merge, and a second image restore. ### Release note Routine Load now persists ALTERed load clauses in the effective origin SQL used during FE recovery. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.load.routineload.RoutineLoadJobPersistenceTest - ./run-fe-ut.sh --run org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest - Docker regression case added but not run locally - Behavior changed: Yes, ALTERed Routine Load definitions survive journal replay and image recovery - Does this need documentation: Yes, the existing design document and PR description must be updated --- .../apache/doris/load/RoutineLoadDesc.java | 2 + .../load/routineload/RoutineLoadJob.java | 6 +-- .../RoutineLoadJobPersistenceTest.java | 37 +++++-------------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java index 862b75cbd3ddef..a035deb488e249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java @@ -108,6 +108,8 @@ public boolean hasSequenceCol() { */ public String toSql() { List clauses = new ArrayList<>(); + // Routine Load SQL does not currently expose a line-delimiter clause. RoutineLoadJob + // persists that scalar separately until the grammar supports representing it here. if (columnSeparator != null) { clauses.add("COLUMNS TERMINATED BY " + SqlUtils.quoteStringLiteral( columnSeparator.getOriSeparator(), SqlModeHelper.hasNoBackSlashEscapes())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 589bb6331c641d..d393a30467b3f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -119,6 +119,7 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); + public static final long DEFAULT_MAX_ERROR_NUM = 0; public static final double DEFAULT_MAX_FILTER_RATIO = 1.0; @@ -204,7 +205,6 @@ public boolean isFinalState() { @SerializedName("men") protected long maxErrorNum = DEFAULT_MAX_ERROR_NUM; // optional protected double maxFilterRatio = DEFAULT_MAX_FILTER_RATIO; - @SerializedName("eml") protected long execMemLimit = DEFAULT_EXEC_MEM_LIMIT; protected int sendBatchParallelism = DEFAULT_SEND_BATCH_PARALLELISM; protected boolean loadToSingleTablet = DEFAULT_LOAD_TO_SINGLE_TABLET; @@ -274,7 +274,7 @@ public boolean isFinalState() { protected String comment = ""; protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); - protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; + protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete protected Expr deleteCondition; // TODO(ml): error sample @@ -320,7 +320,6 @@ public RoutineLoadJob(Long id, String name, this.tableId = tableId; this.authCode = 0; this.userIdentity = userIdentity; - this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -350,7 +349,6 @@ public RoutineLoadJob(Long id, String name, this.authCode = 0; this.userIdentity = userIdentity; this.isMultiTable = true; - this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index b315c93ddd583a..e84bf4184a0d54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -17,19 +17,12 @@ package org.apache.doris.load.routineload; -import org.apache.doris.analysis.ArithmeticExpr; -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ImportColumnDesc; -import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.Separator; -import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Type; import org.apache.doris.common.io.Text; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -54,7 +47,6 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Base64; -import java.util.List; import java.util.Optional; public class RoutineLoadJobPersistenceTest { @@ -81,10 +73,9 @@ public void testImageRestoresLoadDefinitionFromOrigStmt() throws Exception { JsonObject json = imageJson(job); Assert.assertTrue(json.has("ostmt")); - Assert.assertTrue(json.has("eml")); Assert.assertTrue(json.has("mosn")); Assert.assertTrue(json.has("lidel")); - for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "sc", "mt", "dc")) { + for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "sc", "mt", "dc", "eml")) { Assert.assertFalse("load definition must only be persisted through origStmt: " + key, json.has(key)); } @@ -109,17 +100,18 @@ public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exceptio KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "alter_job", 8001L, 9001L, "127.0.0.1:9092", "alter_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.alter_job ON current_table " + job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.alter_job ON current_table WITH MERGE " + "COLUMNS TERMINATED BY ',', " + "COLUMNS(source_col, mapped_col = source_col + 1), " - + "PRECEDING FILTER source_col > 1, WHERE mapped_col < 100, ORDER BY seq_col " + + "PRECEDING FILTER source_col > 1, WHERE mapped_col < 100, " + + "PARTITION(p1), DELETE ON delete_flag = 1, ORDER BY seq_col " + "PROPERTIES (\"exec_mem_limit\" = \"268435456\") " + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " + "\"kafka_topic\" = \"alter_topic\")", 0); job.execMemLimit = 268435456L; - job.setRoutineLoadDesc(initialLoadDesc()); try (MockedStatic ignored = mockCatalog()) { + job = (KafkaRoutineLoadJob) imageRoundTrip(job); job.replayLoadDefinition(new OriginStatement( "ALTER ROUTINE LOAD FOR alter_job COLUMNS TERMINATED BY '|', WHERE mapped_col < 50", 0)); job.replayLoadDefinition(new OriginStatement( @@ -133,7 +125,10 @@ public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exceptio Assert.assertTrue(job.origStmt.originStmt.contains("WHERE")); Assert.assertTrue(job.origStmt.originStmt.contains("PRECEDING FILTER")); Assert.assertTrue(job.origStmt.originStmt.contains("USING ANALYZER")); + Assert.assertTrue(job.origStmt.originStmt.contains("PARTITION(`p1`)")); + Assert.assertTrue(job.origStmt.originStmt.contains("DELETE ON")); Assert.assertTrue(job.origStmt.originStmt.contains("ORDER BY `seq_col`")); + Assert.assertTrue(job.origStmt.originStmt.contains("WITH MERGE")); JsonObject expectedProperties = JsonParser.parseString(job.jobPropertiesToJsonString()).getAsJsonObject(); RoutineLoadJob restored; @@ -142,7 +137,7 @@ public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exceptio } JsonObject restoredProperties = JsonParser.parseString(restored.jobPropertiesToJsonString()).getAsJsonObject(); for (String key : Lists.newArrayList("column_separator", "precedingFilter", - "whereExpr", "sequence_col", "merge_type", "exec_mem_limit")) { + "whereExpr", "partitions", "delete", "sequence_col", "merge_type", "exec_mem_limit")) { Assert.assertEquals(key, expectedProperties.get(key), restoredProperties.get(key)); } Assert.assertTrue(restoredProperties.get("columnToColumnExpr").getAsString().contains("mapped_col=")); @@ -172,20 +167,6 @@ public void testLegacyImageContinuesToRestoreFromOrigStmt() throws Exception { Assert.assertFalse(newImage.has("cs")); } - private static RoutineLoadDesc initialLoadDesc() { - List columns = Lists.newArrayList( - new ImportColumnDesc("source_col"), - new ImportColumnDesc("mapped_col", new ArithmeticExpr(ArithmeticExpr.Operator.ADD, - new SlotRef(null, "source_col"), new IntLiteral(1L), - Type.BIGINT, NullableMode.ALWAYS_NOT_NULLABLE, false))); - Expr preceding = new BinaryPredicate(BinaryPredicate.Operator.GT, - new SlotRef(null, "source_col"), new IntLiteral(1L)); - Expr where = new BinaryPredicate(BinaryPredicate.Operator.LT, - new SlotRef(null, "mapped_col"), new IntLiteral(100L)); - return new RoutineLoadDesc(new Separator(",", ","), null, columns, - preceding, where, null, null, LoadTask.MergeType.APPEND, "seq_col"); - } - private static Separator analyzedSeparator(String value) throws Exception { Separator separator = new Separator(value); separator.analyze(); From 9fb93e1484c27384aa4ce858771570f0e1411797 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 14:46:35 +0800 Subject: [PATCH 08/11] [refactor](routineload) Narrow persistence fix to origin SQL ### What problem does this PR solve? Issue Number: close #66633 Related PR: #66634 Problem Summary: Routine Load images already persist and replay origStmt. The persistence bug is that ALTER load clauses changed runtime fields without updating that statement. Keep the existing gsonPostProcess recovery path unchanged, persist the original ALTER SQL in the journal, and rewrite origStmt to a complete effective CREATE statement after leader and follower ALTER application. Remove direct-field persistence, cache hydration, CSV validation, and other adjacent changes from this PR. ### Release note Routine Load now preserves ALTERed load clauses across follower replay, checkpoints, and FE restart by maintaining the effective CREATE statement. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.load.routineload.RoutineLoadJobPersistenceTest - ./run-fe-ut.sh --run org.apache.doris.load.routineload.KinesisRoutineLoadJobTest - KafkaRoutineLoadJobTest and AlterRoutineLoadOperationLogTest passed in the combined targeted run - Docker regression case added but not run locally - Behavior changed: Yes, ALTERed load clauses update the persisted origin statement - Does this need documentation: Yes, document mixed-version ALTER limitations --- .../org/apache/doris/analysis/Separator.java | 3 - .../apache/doris/load/RoutineLoadDesc.java | 3 +- .../load/routineload/RoutineLoadJob.java | 210 ++++++------------ .../kafka/KafkaRoutineLoadJob.java | 14 +- .../kinesis/KinesisRoutineLoadJob.java | 14 +- .../commands/info/CreateRoutineLoadInfo.java | 11 - .../routineload/KafkaRoutineLoadJobTest.java | 30 +-- .../KinesisRoutineLoadJobTest.java | 37 ++- .../RoutineLoadJobPersistenceTest.java | 29 +-- ...ne_load_alter_checkpoint_restart_fe.groovy | 1 - 10 files changed, 122 insertions(+), 230 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java index 7da2e092a212ad..67515eaca5c79f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java @@ -20,16 +20,13 @@ import org.apache.doris.common.AnalysisException; import com.google.common.base.Strings; -import com.google.gson.annotations.SerializedName; import java.io.StringWriter; public class Separator { private static final String HEX_STRING = "0123456789ABCDEF"; - @SerializedName("os") private final String oriSeparator; - @SerializedName("s") private String separator; public Separator(String separator) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java index a035deb488e249..a357a7b7c3a1c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java @@ -108,8 +108,7 @@ public boolean hasSequenceCol() { */ public String toSql() { List clauses = new ArrayList<>(); - // Routine Load SQL does not currently expose a line-delimiter clause. RoutineLoadJob - // persists that scalar separately until the grammar supports representing it here. + // Routine Load SQL does not currently expose a line-delimiter clause. if (columnSeparator != null) { clauses.add("COLUMNS TERMINATED BY " + SqlUtils.quoteStringLiteral( columnSeparator.getOriSeparator(), SqlModeHelper.hasNoBackSlashEscapes())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index d393a30467b3f9..55888da7806318 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -190,7 +190,6 @@ public boolean isFinalState() { protected Expr precedingFilter; // optional protected Expr whereExpr; // optional protected Separator columnSeparator; // optional - @SerializedName("lidel") protected Separator lineDelimiter; @SerializedName("dtcn") protected int desireTaskConcurrentNum; // optional @@ -236,7 +235,6 @@ public boolean isFinalState() { protected String sequenceCol; - @SerializedName("mosn") protected boolean memtableOnSinkNode = false; protected int currentTaskConcurrentNum; @@ -1971,80 +1969,79 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } + // Process UNIQUE_KEY_UPDATE_MODE first to ensure correct backward compatibility + // with PARTIAL_COLUMNS (HashMap iteration order is not guaranteed) + if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + String modeValue = jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode(modeValue); + if (mode != null) { + uniqueKeyUpdateMode = mode; + isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); + } else { + uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; + } + } + // Process remaining properties + jobProperties.forEach((k, v) -> { + if (k.equals(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + // Backward compatibility: only use partial_columns if unique_key_update_mode is not set + // unique_key_update_mode takes precedence + if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { + isPartialUpdate = Boolean.parseBoolean(v); + if (isPartialUpdate) { + uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + } + } else if (k.equals(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + if ("ERROR".equalsIgnoreCase(v)) { + partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; + } else { + partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; + } + } + }); try { - hydrateJobProperties(); - restoreLoadDefinition(); + ConnectContext ctx = new ConnectContext(); + ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); + StatementContext statementContext = new StatementContext(); + statementContext.setConnectContext(ctx); + ctx.setStatementContext(statementContext); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.getState().reset(); + try { + ctx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( + origStmt.originStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + // If tableId is set, resolve the current table name by ID so that + // table rename / SWAP TABLE won't cause replay to fail with stale name in origStmt. + if (!isMultiTable && tableId != 0) { + try { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); + if (db != null) { + db.getTable(tableId).ifPresent( + table -> createRoutineLoadInfo.setTableName(table.getName())); + } + } catch (Exception ignored) { + // fall through; let validate() surface the real error + } + } + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); + } finally { + ctx.cleanup(); + } } catch (Exception e) { this.state = JobState.CANCELLED; - LOG.warn("error happens when restoring routine load job", e); + LOG.warn("error happens when parsing create routine load stmt: " + origStmt.originStmt, e); } if (userIdentity != null) { userIdentity.setIsAnalyzed(); } } - private void hydrateJobProperties() throws UserException { - if (jobProperties.containsKey(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)) { - maxFilterRatio = Double.parseDouble( - jobProperties.get(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)) { - sendBatchParallelism = Integer.parseInt( - jobProperties.get(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)) { - loadToSingleTablet = Boolean.parseBoolean( - jobProperties.get(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)); - } - - boolean hasUniqueKeyUpdateMode = jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - if (hasUniqueKeyUpdateMode) { - TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode( - jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - uniqueKeyUpdateMode = mode == null ? TUniqueKeyUpdateMode.UPSERT : mode; - isPartialUpdate = uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - if (!hasUniqueKeyUpdateMode && jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - isPartialUpdate = Boolean.parseBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - if (isPartialUpdate) { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase( - jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) - ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; - } - - if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { - enclose = parseEnclose(jobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); - } - if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { - escape = parseEscape(jobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); - } - if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { - emptyFieldAsNull = Boolean.parseBoolean( - jobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); - } - } - - private void restoreLoadDefinition() throws UserException { - Database database = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); - ConnectContext ctx = createLoadDefinitionContext(database); - try { - ctx.setThreadLocalInfo(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) parsePersistedStatement(origStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - String currentTableName = isMultiTable ? "" : getTableName(); - setRoutineLoadDesc(createRoutineLoadInfo.analyzeLoadProperties( - ctx, database.getName(), currentTableName)); - createRoutineLoadInfo.checkJobProperties(); - execMemLimit = createRoutineLoadInfo.getExecMemLimit(); - } finally { - ctx.cleanup(); - } - } - protected void replayLoadDefinition(OriginStatement alterStatement) throws UserException { if (alterStatement == null) { return; @@ -2091,7 +2088,7 @@ protected void mergeLoadDescToOriginStatement() throws UserException { if (!loadClauseSql.isEmpty()) { sql.append(" ").append(loadClauseSql); } - sql.append(" PROPERTIES (\"exec_mem_limit\" = \"").append(execMemLimit).append("\")"); + sql.append(" PROPERTIES (\"desired_concurrent_number\" = \"1\")"); sql.append(buildPersistedDataSourceSql()); origStmt = new OriginStatement(sql.toString(), 0); } @@ -2115,26 +2112,7 @@ private LogicalPlan parsePersistedStatement(OriginStatement statement) { public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; - // Leader-only validation. Replay must accept values written by older FE versions. - protected void validateCommonJobProperties(Map jobProperties) throws UserException { - validateCsvFormatProperties(jobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( - jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); - } - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if (!"APPEND".equalsIgnoreCase(policy) && !"ERROR".equalsIgnoreCase(policy)) { - throw new AnalysisException(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY - + " should be one of {'APPEND', 'ERROR'}, but found " + policy); - } - } - } - - // Apply ALTER ROUTINE LOAD properties. The leader validates before mutation; replay trusts the journal. + // for ALTER ROUTINE LOAD protected void modifyCommonJobProperties(Map jobProperties) throws UserException { if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( @@ -2169,7 +2147,12 @@ protected void modifyCommonJobProperties(Map jobProperties) thro if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - this.uniqueKeyUpdateMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); + TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); + // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } + this.uniqueKeyUpdateMode = newMode; this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); @@ -2186,55 +2169,6 @@ protected void modifyCommonJobProperties(Map jobProperties) thro this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); } - - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase(policy) - ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; - this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, - partialUpdateNewKeyPolicy.name()); - } - - if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { - String value = jobProperties.remove(CsvFileFormatProperties.PROP_ENCLOSE); - enclose = parseEnclose(value); - this.jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, value); - } - if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { - String value = jobProperties.remove(CsvFileFormatProperties.PROP_ESCAPE); - escape = parseEscape(value); - this.jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, value); - } - if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { - String value = jobProperties.remove(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL); - emptyFieldAsNull = Boolean.parseBoolean(value); - this.jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, value); - } - } - - private static void validateCsvFormatProperties(Map jobProperties) { - if (!jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE) - && !jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE) - && !jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { - return; - } - Map csvProperties = Maps.newHashMap(); - for (String property : new String[] {CsvFileFormatProperties.PROP_ENCLOSE, - CsvFileFormatProperties.PROP_ESCAPE, CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL}) { - if (jobProperties.containsKey(property)) { - csvProperties.put(property, jobProperties.get(property)); - } - } - CsvFileFormatProperties properties = new CsvFileFormatProperties(FileFormatProperties.FORMAT_CSV); - properties.analyzeFileFormatProperties(csvProperties, false); - } - - private static byte parseEnclose(String value) { - return Strings.isNullOrEmpty(value) ? 0 : (byte) value.charAt(0); - } - - private static byte parseEscape(String value) { - return Strings.isNullOrEmpty(value) ? 0 : value.getBytes()[0]; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index 100d4692bcefa8..bfd7d617e89fc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -60,6 +60,7 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -73,6 +74,7 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -776,7 +778,6 @@ public Map getCustomProperties() { @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); - validateCommonJobProperties(jobProperties); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); if (null != dataSourceProperties) { // if the partition offset is set by timestamp, convert it to real offset @@ -882,6 +883,17 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + if ("ERROR".equalsIgnoreCase(policy)) { + this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; + } else { + this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; + } + } } LOG.info("modify the properties of kafka routine load job: {}, jobProperties: {}, datasource properties: {}", this.id, jobProperties, dataSourceProperties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 146c514e67b6a0..80545623170559 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -51,6 +51,7 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -64,6 +65,7 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -684,7 +686,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti throw new DdlException("Only supports modification of PAUSED jobs"); } - validateCommonJobProperties(jobProperties); modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); mergeLoadDescToOriginStatement(); @@ -763,6 +764,17 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + if ("ERROR".equalsIgnoreCase(policy)) { + this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; + } else { + this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; + } + } } LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", this.id, jobProperties, dataSourceProperties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java index a24d1a72980c27..e8a75c0299b4d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateRoutineLoadInfo.java @@ -412,17 +412,6 @@ public void validate(ConnectContext ctx) throws UserException { } } - /** - * Analyze only the load clauses from a persisted CREATE statement. RoutineLoadJob uses this - * during image and journal replay because job and data-source properties have their own - * persisted state. - */ - public RoutineLoadDesc analyzeLoadProperties(ConnectContext ctx, String currentDbName, - String currentTableName) throws UserException { - return checkLoadProperties(ctx, loadPropertyMap, currentDbName, currentTableName, - isMultiTable, mergeType); - } - private void checkDBTable(ConnectContext ctx) throws AnalysisException { labelNameInfo.validate(ctx); dbName = labelNameInfo.getDb(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 321566589770e7..56dee6daaea93b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -33,7 +33,6 @@ import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; -import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; @@ -286,7 +285,7 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro } @Test - public void testAlterPersistsOriginStatementAndCsvPropertiesForReplay() throws Exception { + public void testAlterPersistsOriginStatementForReplay() throws Exception { KafkaRoutineLoadJob leader = createPausedJob(); KafkaRoutineLoadJob follower = createPausedJob(); RoutineLoadDesc originalDesc = new RoutineLoadDesc(new Separator("|", "|"), null, null, @@ -297,9 +296,6 @@ public void testAlterPersistsOriginStatementAndCsvPropertiesForReplay() throws E follower.origStmt = initialOriginStatement(); Map jobProperties = Maps.newHashMap(); - jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); - jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); - jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, null); OriginStatement alterStatement = new OriginStatement( @@ -353,21 +349,6 @@ public void testAlterPersistsOriginStatementAndCsvPropertiesForReplay() throws E } } - @Test - public void testReplayLegacyCsvPropertiesDoesNotRunNewValidation() { - KafkaRoutineLoadJob follower = createPausedJob(); - Map legacyJobProperties = Maps.newHashMap(); - legacyJobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "legacy"); - AlterRoutineLoadJobOperationLog legacyLog = new AlterRoutineLoadJobOperationLog( - follower.getId(), legacyJobProperties, null); - - follower.replayModifyProperties(legacyLog); - - Assert.assertEquals((byte) 'l', follower.getEnclose()); - Map persistedJobProperties = Deencapsulation.getField(follower, "jobProperties"); - Assert.assertEquals("legacy", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); - } - private static KafkaRoutineLoadJob createPausedJob() { KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1L, "job1", 1L, 1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN); @@ -379,15 +360,6 @@ private static void assertAlterState(RoutineLoadJob job) { Assert.assertEquals(";", job.getColumnSeparator().getSeparator()); Assert.assertNull(job.getLineDelimiter()); Assert.assertEquals("original_sequence", job.getSequenceCol()); - Assert.assertEquals((byte) '"', job.getEnclose()); - Assert.assertEquals((byte) '\\', job.getEscape()); - Assert.assertTrue(job.getEmptyFieldAsNull()); - Assert.assertEquals(Boolean.TRUE, Deencapsulation.getField(job, "emptyFieldAsNull")); - - Map persistedJobProperties = Deencapsulation.getField(job, "jobProperties"); - Assert.assertEquals("\"", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); - Assert.assertEquals("\\", persistedJobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); - Assert.assertEquals("true", persistedJobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } private static OriginStatement initialOriginStatement() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 4ef10196f66d29..8483f006998598 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -27,7 +27,6 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; @@ -35,7 +34,6 @@ import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; @@ -254,14 +252,11 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi } @Test - public void testAlterOriginStatementReplayKeepsCsvCachesInCheckpointParity() throws Exception { + public void testAlterOriginStatementReplayKeepsCheckpointParity() throws Exception { KinesisRoutineLoadJob leader = createPausedJobWithInitialLoadDesc(); KinesisRoutineLoadJob replay = createPausedJobWithInitialLoadDesc(); Map jobProperties = Maps.newHashMap(); - jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); - jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); - jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, "sequence_col"); OriginStatement alterStatement = new OriginStatement( @@ -308,24 +303,11 @@ public void testAlterOriginStatementReplayKeepsCsvCachesInCheckpointParity() thr assertAlterResult(replay); Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), JsonParser.parseString(checkpointJson(replay))); + assertAlterResult(imageRoundTrip(leader)); + assertAlterResult(imageRoundTrip(replay)); } } - @Test - public void testAlterValidatesCsvBeforeDataSourceMutation() { - KinesisRoutineLoadJob job = createPausedJobWithInitialLoadDesc(); - Map jobProperties = Maps.newHashMap(); - jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); - KinesisDataSourceProperties dataSourceProperties = Mockito.mock(KinesisDataSourceProperties.class); - AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); - Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); - Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); - - Assert.assertThrows(AnalysisException.class, () -> job.modifyProperties(command)); - Assert.assertEquals("stream-1", job.getStream()); - Mockito.verifyNoInteractions(dataSourceProperties); - } - @Test public void testShardRefreshShouldMoveRetiredParentToClosedUntilConsumed() throws Exception { KinesisRoutineLoadJob routineLoadJob = @@ -453,9 +435,6 @@ private void assertAlterResult(KinesisRoutineLoadJob job) { Assert.assertEquals(";", job.getColumnSeparator().getSeparator()); Assert.assertNull(job.getLineDelimiter()); Assert.assertEquals("sequence_col", job.getSequenceCol()); - Assert.assertEquals((byte) '"', job.getEnclose()); - Assert.assertEquals((byte) '\\', job.getEscape()); - Assert.assertTrue(job.getEmptyFieldAsNull()); } private AlterRoutineLoadJobOperationLog journalRoundTrip(AlterRoutineLoadJobOperationLog log) @@ -479,6 +458,16 @@ private String checkpointJson(RoutineLoadJob job) throws Exception { } } + private KinesisRoutineLoadJob imageRoundTrip(RoutineLoadJob job) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bytes)) { + job.write(out); + } + try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + return (KinesisRoutineLoadJob) RoutineLoadJob.read(in); + } + } + private Set collectAssignedShards(KinesisRoutineLoadJob routineLoadJob) { List routineLoadTaskInfoList = Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index e84bf4184a0d54..9e3945ea1a777f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -22,7 +22,9 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; import org.apache.doris.common.io.Text; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -62,20 +64,17 @@ public void testImageRestoresLoadDefinitionFromOrigStmt() throws Exception { + "COLUMNS TERMINATED BY '|', " + "COLUMNS(source_col, mapped_col = source_col + 1), " + "PRECEDING FILTER source_col > 1, WHERE mapped_col <= 10 " - + "PROPERTIES (\"exec_mem_limit\" = \"345678901\") " + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " + "\"kafka_topic\" = \"image_topic\")", 0); - job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator(",", ","), analyzedSeparator("\\n"), + job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator(",", ","), null, Lists.newArrayList(new ImportColumnDesc("wrong_column")), null, null, null, null, LoadTask.MergeType.APPEND, null)); - job.memtableOnSinkNode = true; JsonObject json = imageJson(job); Assert.assertTrue(json.has("ostmt")); - Assert.assertTrue(json.has("mosn")); - Assert.assertTrue(json.has("lidel")); - for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "sc", "mt", "dc", "eml")) { + for (String key : Lists.newArrayList( + "pni", "cds", "pf", "we", "cs", "sc", "mt", "dc", "eml", "mosn", "lidel")) { Assert.assertFalse("load definition must only be persisted through origStmt: " + key, json.has(key)); } @@ -90,9 +89,6 @@ public void testImageRestoresLoadDefinitionFromOrigStmt() throws Exception { Assert.assertEquals("mapped_col", restored.getColumnExprDescs().descs.get(1).getColumnName()); Assert.assertNotNull(restored.getPrecedingFilter()); Assert.assertNotNull(restored.getWhereExpr()); - Assert.assertEquals(345678901L, restored.getMemLimit()); - Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); - Assert.assertTrue(restored.isMemtableOnSinkNode()); } @Test @@ -105,10 +101,8 @@ public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exceptio + "COLUMNS(source_col, mapped_col = source_col + 1), " + "PRECEDING FILTER source_col > 1, WHERE mapped_col < 100, " + "PARTITION(p1), DELETE ON delete_flag = 1, ORDER BY seq_col " - + "PROPERTIES (\"exec_mem_limit\" = \"268435456\") " + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " + "\"kafka_topic\" = \"alter_topic\")", 0); - job.execMemLimit = 268435456L; try (MockedStatic ignored = mockCatalog()) { job = (KafkaRoutineLoadJob) imageRoundTrip(job); @@ -137,7 +131,7 @@ public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exceptio } JsonObject restoredProperties = JsonParser.parseString(restored.jobPropertiesToJsonString()).getAsJsonObject(); for (String key : Lists.newArrayList("column_separator", "precedingFilter", - "whereExpr", "partitions", "delete", "sequence_col", "merge_type", "exec_mem_limit")) { + "whereExpr", "partitions", "delete", "sequence_col", "merge_type")) { Assert.assertEquals(key, expectedProperties.get(key), restoredProperties.get(key)); } Assert.assertTrue(restoredProperties.get("columnToColumnExpr").getAsString().contains("mapped_col=")); @@ -158,8 +152,6 @@ public void testLegacyImageContinuesToRestoreFromOrigStmt() throws Exception { Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); - Assert.assertEquals(33554432L, restored.getMemLimit()); - Assert.assertFalse(restored.isMemtableOnSinkNode()); JsonObject newImage = imageJson(restored); Assert.assertTrue(newImage.has("ostmt")); @@ -167,12 +159,6 @@ public void testLegacyImageContinuesToRestoreFromOrigStmt() throws Exception { Assert.assertFalse(newImage.has("cs")); } - private static Separator analyzedSeparator(String value) throws Exception { - Separator separator = new Separator(value); - separator.analyze(); - return separator; - } - private static MockedStatic mockCatalog() throws Exception { Env env = Mockito.mock(Env.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -191,6 +177,9 @@ private static MockedStatic mockCatalog() throws Exception { Mockito.when(database.getTableOrMetaException(9001L)).thenReturn(table); Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); Mockito.when(table.getName()).thenReturn("current_table"); + Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); + Mockito.when(table.getKeysType()).thenReturn(KeysType.UNIQUE_KEYS); + Mockito.when(table.hasDeleteSign()).thenReturn(true); Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); MockedStatic envStatic = Mockito.mockStatic(Env.class); diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy index a40a47993608e4..0045c740660eca 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy @@ -45,7 +45,6 @@ suite("test_routine_load_alter_checkpoint_restart_fe", "docker") { "column_separator", "precedingFilter", "whereExpr", - "exec_mem_limit", "merge_type" ] From 23de67ba316b8612c5853db11e30ae7412e2d77f Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 15:12:39 +0800 Subject: [PATCH 09/11] [refactor](routineload) Restore direct load definition persistence ### What problem does this PR solve? Issue Number: close #66633 Related PR: #66634 Problem Summary: Treat the effective Routine Load fields as authoritative metadata instead of rewriting origStmt after ALTER. Persist the load-definition fields directly in images, persist RoutineLoadDesc deltas in ALTER journals, and use the original CREATE statement only when reading legacy images whose nullable effective fields are absent. Empty new definitions may also use the fallback safely because ALTER cannot unset all load clauses. ### Release note Routine Load now preserves ALTERed load clauses across journal replay and FE restart through direct metadata persistence. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.load.routineload.RoutineLoadJobPersistenceTest,org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest - Behavior changed: Yes, image and ALTER journal persist effective Routine Load definitions directly - Does this need documentation: Yes, document mixed-version ALTER limitations --- .../org/apache/doris/analysis/Separator.java | 3 + .../apache/doris/load/RoutineLoadDesc.java | 62 ++------- .../load/routineload/RoutineLoadJob.java | 89 ++++--------- .../kafka/KafkaRoutineLoadJob.java | 5 +- .../kinesis/KinesisRoutineLoadJob.java | 5 +- .../commands/AlterRoutineLoadCommand.java | 13 -- .../AlterRoutineLoadJobOperationLog.java | 14 +- .../routineload/KafkaRoutineLoadJobTest.java | 7 +- .../KinesisRoutineLoadJobTest.java | 8 +- .../RoutineLoadJobPersistenceTest.java | 122 +++++++++--------- .../AlterRoutineLoadOperationLogTest.java | 39 ++++-- .../routine-load/a8928245/PROVENANCE.txt | 2 +- 12 files changed, 146 insertions(+), 223 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java index 67515eaca5c79f..7da2e092a212ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Separator.java @@ -20,13 +20,16 @@ import org.apache.doris.common.AnalysisException; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; import java.io.StringWriter; public class Separator { private static final String HEX_STRING = "0123456789ABCDEF"; + @SerializedName("os") private final String oriSeparator; + @SerializedName("s") private String separator; public Separator(String separator) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java index a357a7b7c3a1c6..28a429960d2b71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/RoutineLoadDesc.java @@ -18,33 +18,37 @@ package org.apache.doris.load; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; -import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.SqlUtils; import org.apache.doris.load.loadv2.LoadTask; -import org.apache.doris.qe.SqlModeHelper; import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class RoutineLoadDesc { + @SerializedName("cs") private final Separator columnSeparator; + @SerializedName("ld") private final Separator lineDelimiter; + @SerializedName("cols") private final List columnsInfo; + @SerializedName("pf") private final Expr precedingFilter; + @SerializedName("f") private final Expr filter; + @SerializedName("dc") private final Expr deleteCondition; + @SerializedName("mt") private LoadTask.MergeType mergeType; // nullable + @SerializedName("pn") private final PartitionNamesInfo partitionNamesInfo; + @SerializedName("sc") private final String sequenceColName; public RoutineLoadDesc(Separator columnSeparator, Separator lineDelimiter, List columnsInfo, @@ -103,52 +107,6 @@ public boolean hasSequenceCol() { return !Strings.isNullOrEmpty(sequenceColName); } - /** - * Convert the effective load clauses to SQL so they can be persisted in RoutineLoadJob.origStmt. - */ - public String toSql() { - List clauses = new ArrayList<>(); - // Routine Load SQL does not currently expose a line-delimiter clause. - if (columnSeparator != null) { - clauses.add("COLUMNS TERMINATED BY " + SqlUtils.quoteStringLiteral( - columnSeparator.getOriSeparator(), SqlModeHelper.hasNoBackSlashEscapes())); - } - if (columnsInfo != null) { - clauses.add("COLUMNS(" + columnsInfo.stream() - .map(this::columnToSql) - .collect(Collectors.joining(", ")) + ")"); - } - if (precedingFilter != null) { - clauses.add("PRECEDING FILTER " + precedingFilter.accept( - ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); - } - if (filter != null) { - clauses.add("WHERE " + filter.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); - } - if (partitionNamesInfo != null) { - String prefix = partitionNamesInfo.isTemp() ? "TEMPORARY PARTITION(" : "PARTITION("; - clauses.add(prefix + partitionNamesInfo.getPartitionNames().stream() - .map(SqlUtils::getIdentSql) - .collect(Collectors.joining(", ")) + ")"); - } - if (deleteCondition != null) { - clauses.add("DELETE ON " + deleteCondition.accept( - ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); - } - if (hasSequenceCol()) { - clauses.add("ORDER BY " + SqlUtils.getIdentSql(sequenceColName)); - } - return String.join(", ", clauses); - } - - private String columnToSql(ImportColumnDesc columnDesc) { - String sql = SqlUtils.getIdentSql(columnDesc.getColumnName()); - if (columnDesc.getExpr() != null) { - sql += " = " + columnDesc.getExpr().accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); - } - return sql; - } - public void analyze() throws UserException { if (mergeType != LoadTask.MergeType.MERGE && deleteCondition != null) { throw new AnalysisException("not support DELETE ON clause when merge type is not MERGE."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 55888da7806318..a30b1cbd2ff109 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.ExprToSqlVisitor; -import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; @@ -42,7 +41,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; -import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; @@ -56,7 +54,6 @@ import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; import org.apache.doris.nereids.load.NereidsStreamLoadPlanner; import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.nereids.trees.plans.commands.LoadCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; @@ -185,11 +182,17 @@ public boolean isFinalState() { // this code is used to verify be task request protected long authCode; // protected RoutineLoadDesc routineLoadDesc; // optional + @SerializedName("pni") protected PartitionNamesInfo partitionNamesInfo; // optional + @SerializedName("cds") protected ImportColumnDescs columnDescs; // optional + @SerializedName("pf") protected Expr precedingFilter; // optional + @SerializedName("we") protected Expr whereExpr; // optional + @SerializedName("cs") protected Separator columnSeparator; // optional + @SerializedName("lidel") protected Separator lineDelimiter; @SerializedName("dtcn") protected int desireTaskConcurrentNum; // optional @@ -233,6 +236,7 @@ public boolean isFinalState() { protected TPartialUpdateNewRowPolicy partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; protected TUniqueKeyUpdateMode uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; + @SerializedName("sc") protected String sequenceCol; protected boolean memtableOnSinkNode = false; @@ -261,7 +265,7 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // Persist the current effective load definition as a CREATE statement. + // Keep the original CREATE statement for downgrade compatibility and legacy image recovery. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -272,7 +276,9 @@ public boolean isFinalState() { protected String comment = ""; protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); + @SerializedName("mt") protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete + @SerializedName("dc") protected Expr deleteCondition; // TODO(ml): error sample @@ -2000,6 +2006,15 @@ public void gsonPostProcess() throws IOException { } } }); + // Old images have none of the nullable effective load-definition fields. A new image with + // an empty definition also takes this path, which is safe because ALTER cannot unset all + // load clauses and the original CREATE statement therefore has the same empty definition. + if (hasPersistedLoadDefinition()) { + if (userIdentity != null) { + userIdentity.setIsAnalyzed(); + } + return; + } try { ConnectContext ctx = new ConnectContext(); ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); @@ -2042,68 +2057,10 @@ public void gsonPostProcess() throws IOException { } } - protected void replayLoadDefinition(OriginStatement alterStatement) throws UserException { - if (alterStatement == null) { - return; - } - Database database = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get(); - ConnectContext ctx = createLoadDefinitionContext(database); - try { - ctx.setThreadLocalInfo(); - AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) parsePersistedStatement(alterStatement); - setRoutineLoadDesc(command.analyzeLoadProperties(ctx, this)); - mergeLoadDescToOriginStatement(); - } finally { - ctx.cleanup(); - } - } - - private ConnectContext createLoadDefinitionContext(Database database) { - ConnectContext ctx = new ConnectContext(); - ctx.setDatabase(database.getName()); - StatementContext statementContext = new StatementContext(); - statementContext.setConnectContext(ctx); - ctx.setStatementContext(statementContext); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setCurrentUserIdentity(UserIdentity.ADMIN); - if (sessionVariables.containsKey(SessionVariable.SQL_MODE)) { - ctx.getSessionVariable().setSqlMode(Long.parseLong(sessionVariables.get(SessionVariable.SQL_MODE))); - } - ctx.getState().reset(); - return ctx; - } - - protected void mergeLoadDescToOriginStatement() throws UserException { - List columns = - columnDescs == null ? null : Lists.newArrayList(columnDescs.descs); - RoutineLoadDesc loadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, - precedingFilter, whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol); - StringBuilder sql = new StringBuilder("CREATE ROUTINE LOAD ") - .append(SqlUtils.getIdentSql(name)); - if (!isMultiTable) { - sql.append(" ON ").append(SqlUtils.getIdentSql(getTableName())); - } - sql.append(" WITH ").append(mergeType.name()); - String loadClauseSql = loadDesc.toSql(); - if (!loadClauseSql.isEmpty()) { - sql.append(" ").append(loadClauseSql); - } - sql.append(" PROPERTIES (\"desired_concurrent_number\" = \"1\")"); - sql.append(buildPersistedDataSourceSql()); - origStmt = new OriginStatement(sql.toString(), 0); - } - - private String buildPersistedDataSourceSql() { - if (dataSourceType == LoadDataSourceType.KINESIS) { - return " FROM KINESIS (\"aws.region\" = \"us-east-1\", " - + "\"kinesis_stream\" = \"__routine_load_persistence__\")"; - } - return " FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " - + "\"kafka_topic\" = \"__routine_load_persistence__\")"; - } - - private LogicalPlan parsePersistedStatement(OriginStatement statement) { - return new NereidsParser().parseMultiple(statement.originStmt).get(statement.idx).first; + private boolean hasPersistedLoadDefinition() { + return partitionNamesInfo != null || columnDescs != null || precedingFilter != null + || whereExpr != null || columnSeparator != null || lineDelimiter != null + || sequenceCol != null || deleteCondition != null; } public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index bfd7d617e89fc6..da15d026182147 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -792,10 +792,9 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); - mergeLoadDescToOriginStatement(); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getOriginStatement()); + jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -922,7 +921,7 @@ private void resetCloudProgress(Cloud.ResetRLProgressRequest.Builder builder) th public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KafkaDataSourceProperties) log.getDataSourceProperties()); - replayLoadDefinition(log.getOriginStatement()); + setRoutineLoadDesc(log.getRoutineLoadDesc()); } catch (UserException e) { // should not happen LOG.error("failed to replay modify kafka routine load job: {}", id, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 80545623170559..8eb62601620acb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -688,10 +688,9 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); - mergeLoadDescToOriginStatement(); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id, - jobProperties, dataSourceProperties, command.getOriginStatement()); + jobProperties, dataSourceProperties, command.getRoutineLoadDesc()); Env.getCurrentEnv().getEditLog().logAlterRoutineLoadJob(log); } finally { writeUnlock(); @@ -785,7 +784,7 @@ public void replayModifyProperties(AlterRoutineLoadJobOperationLog log) { try { modifyPropertiesInternal(log.getJobProperties(), (KinesisDataSourceProperties) log.getDataSourceProperties()); - replayLoadDefinition(log.getOriginStatement()); + setRoutineLoadDesc(log.getRoutineLoadDesc()); } catch (UserException e) { LOG.error("failed to replay modify kinesis routine load job: {}", id, e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java index 3b02c598705bab..367480c5d934d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterRoutineLoadCommand.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.trees.plans.commands.load.LoadProperty; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.StmtExecutor; import com.google.common.collect.ImmutableSet; @@ -88,7 +87,6 @@ public class AlterRoutineLoadCommand extends AlterCommand { private final LabelNameInfo labelNameInfo; private final Map loadPropertyMap; private RoutineLoadDesc routineLoadDesc; - private OriginStatement originStatement; private final Map jobProperties; private final Map dataSourceMapProperties; private boolean isPartialUpdate; @@ -151,16 +149,6 @@ public RoutineLoadDesc getRoutineLoadDesc() { return routineLoadDesc; } - public OriginStatement getOriginStatement() { - return originStatement; - } - - /** Analyze only the load-clause delta while replaying the persisted ALTER statement. */ - public RoutineLoadDesc analyzeLoadProperties(ConnectContext ctx, RoutineLoadJob job) throws UserException { - return CreateRoutineLoadInfo.checkLoadProperties(ctx, loadPropertyMap, - job.getDbFullName(), job.getTableName(), job.isMultiTable(), job.getMergeType()); - } - @Override public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { validate(ctx); @@ -171,7 +159,6 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { * validate */ public void validate(ConnectContext ctx) throws UserException { - originStatement = ctx.getStatementContext().getOriginStatement(); labelNameInfo.validate(ctx); FeNameFormat.checkCommonName(NAME_TYPE, labelNameInfo.getLabel()); // check routine load job properties include desired concurrent number etc. diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java index 28a77a49e46277..9d8064943c566a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterRoutineLoadJobOperationLog.java @@ -19,9 +19,9 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.routineload.AbstractDataSourceProperties; import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.qe.OriginStatement; import com.google.gson.annotations.SerializedName; @@ -38,8 +38,8 @@ public class AlterRoutineLoadJobOperationLog implements Writable { private Map jobProperties; @SerializedName(value = "dataSourceProperties") private AbstractDataSourceProperties dataSourceProperties; - @SerializedName(value = "originStatement") - private OriginStatement originStatement; + @SerializedName(value = "routineLoadDesc") + private RoutineLoadDesc routineLoadDesc; public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, AbstractDataSourceProperties dataSourceProperties) { @@ -47,11 +47,11 @@ public AlterRoutineLoadJobOperationLog(long jobId, Map jobProper } public AlterRoutineLoadJobOperationLog(long jobId, Map jobProperties, - AbstractDataSourceProperties dataSourceProperties, OriginStatement originStatement) { + AbstractDataSourceProperties dataSourceProperties, RoutineLoadDesc routineLoadDesc) { this.jobId = jobId; this.jobProperties = jobProperties; this.dataSourceProperties = dataSourceProperties; - this.originStatement = originStatement; + this.routineLoadDesc = routineLoadDesc; } public long getJobId() { @@ -66,8 +66,8 @@ public AbstractDataSourceProperties getDataSourceProperties() { return dataSourceProperties; } - public OriginStatement getOriginStatement() { - return originStatement; + public RoutineLoadDesc getRoutineLoadDesc() { + return routineLoadDesc; } public static AlterRoutineLoadJobOperationLog read(DataInput in) throws IOException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 56dee6daaea93b..7e80969d839c25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -285,7 +285,7 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro } @Test - public void testAlterPersistsOriginStatementForReplay() throws Exception { + public void testAlterPersistsRoutineLoadDescForReplay() throws Exception { KafkaRoutineLoadJob leader = createPausedJob(); KafkaRoutineLoadJob follower = createPausedJob(); RoutineLoadDesc originalDesc = new RoutineLoadDesc(new Separator("|", "|"), null, null, @@ -298,13 +298,10 @@ public void testAlterPersistsOriginStatementForReplay() throws Exception { Map jobProperties = Maps.newHashMap(); RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, null); - OriginStatement alterStatement = new OriginStatement( - "ALTER ROUTINE LOAD FOR job1 COLUMNS TERMINATED BY ';'", 0); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(null); Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); - Mockito.when(command.getOriginStatement()).thenReturn(alterStatement); Env env = Mockito.mock(Env.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -337,7 +334,7 @@ public void testAlterPersistsOriginStatementForReplay() throws Exception { ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); alterLog = logCaptor.getValue(); - Assert.assertEquals(alterStatement.originStmt, alterLog.getOriginStatement().originStmt); + Assert.assertEquals(";", alterLog.getRoutineLoadDesc().getColumnSeparator().getSeparator()); Assert.assertEquals(jobProperties, alterLog.getJobProperties()); assertAlterState(leader); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 8483f006998598..18fad2af04824b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -252,21 +252,17 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi } @Test - public void testAlterOriginStatementReplayKeepsCheckpointParity() throws Exception { + public void testAlterRoutineLoadDescReplayKeepsCheckpointParity() throws Exception { KinesisRoutineLoadJob leader = createPausedJobWithInitialLoadDesc(); KinesisRoutineLoadJob replay = createPausedJobWithInitialLoadDesc(); Map jobProperties = Maps.newHashMap(); RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, "sequence_col"); - OriginStatement alterStatement = new OriginStatement( - "ALTER ROUTINE LOAD FOR kinesis_routine_load_job " - + "COLUMNS TERMINATED BY ';', ORDER BY sequence_col", 0); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); Mockito.when(command.getDataSourceProperties()).thenReturn(null); Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); - Mockito.when(command.getOriginStatement()).thenReturn(alterStatement); Env env = Mockito.mock(Env.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -298,7 +294,7 @@ public void testAlterOriginStatementReplayKeepsCheckpointParity() throws Excepti AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue()); replay.replayModifyProperties(log); - Assert.assertEquals(alterStatement.originStmt, log.getOriginStatement().originStmt); + Assert.assertEquals(";", log.getRoutineLoadDesc().getColumnSeparator().getSeparator()); assertAlterResult(leader); assertAlterResult(replay); Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 9e3945ea1a777f..8ee7baf80ecf87 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -17,14 +17,20 @@ package org.apache.doris.load.routineload; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; @@ -56,86 +62,73 @@ public class RoutineLoadJobPersistenceTest { "/upgrade/routine-load/a8928245/routine-load-kafka-image.b64"; @Test - public void testImageRestoresLoadDefinitionFromOrigStmt() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "image_job", 8001L, + public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "direct_job", 8001L, 9001L, "127.0.0.1:9092", "image_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.image_job ON stale_table " - + "COLUMNS TERMINATED BY '|', " - + "COLUMNS(source_col, mapped_col = source_col + 1), " - + "PRECEDING FILTER source_col > 1, WHERE mapped_col <= 10 " - + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " - + "\"kafka_topic\" = \"image_topic\")", 0); - - job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator(",", ","), null, - Lists.newArrayList(new ImportColumnDesc("wrong_column")), - null, null, null, null, LoadTask.MergeType.APPEND, null)); + job.origStmt = new OriginStatement("deliberately invalid SQL", 0); + Expr columnExpr = new BinaryPredicate( + BinaryPredicate.Operator.GT, new IntLiteral(3), new IntLiteral(2)); + Expr precedingFilter = new BinaryPredicate( + BinaryPredicate.Operator.GE, new IntLiteral(4), new IntLiteral(3)); + Expr whereExpr = new BinaryPredicate( + BinaryPredicate.Operator.LT, new IntLiteral(1), new IntLiteral(2)); + Expr deleteCondition = new BinaryPredicate( + BinaryPredicate.Operator.EQ, new IntLiteral(1), new IntLiteral(1)); + job.setRoutineLoadDesc(new RoutineLoadDesc( + new Separator("|", "|"), new Separator("\n", "\\n"), + Lists.newArrayList(new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", columnExpr)), + precedingFilter, whereExpr, + new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")), + deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); JsonObject json = imageJson(job); Assert.assertTrue(json.has("ostmt")); for (String key : Lists.newArrayList( - "pni", "cds", "pf", "we", "cs", "sc", "mt", "dc", "eml", "mosn", "lidel")) { - Assert.assertFalse("load definition must only be persisted through origStmt: " + key, json.has(key)); + "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc")) { + Assert.assertTrue("missing direct-state key " + key, json.has(key)); } RoutineLoadJob restored; - try (MockedStatic ignored = mockCatalog()) { + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { restored = imageRoundTrip(job); + envStatic.verifyNoInteractions(); } - Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + restored.getPartitionNamesInfo().getPartitionNames()); Assert.assertEquals(2, restored.getColumnExprDescs().descs.size()); - Assert.assertEquals("source_col", restored.getColumnExprDescs().descs.get(0).getColumnName()); - Assert.assertEquals("mapped_col", restored.getColumnExprDescs().descs.get(1).getColumnName()); - Assert.assertNotNull(restored.getPrecedingFilter()); - Assert.assertNotNull(restored.getWhereExpr()); + Assert.assertEquals(exprToSql(columnExpr), + exprToSql(restored.getColumnExprDescs().descs.get(1).getExpr())); + Assert.assertEquals(exprToSql(precedingFilter), exprToSql(restored.getPrecedingFilter())); + Assert.assertEquals(exprToSql(whereExpr), exprToSql(restored.getWhereExpr())); + Assert.assertEquals(exprToSql(deleteCondition), exprToSql(restored.getDeleteCondition())); + Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); + Assert.assertEquals("seq_col", restored.getSequenceCol()); + Assert.assertEquals(LoadTask.MergeType.MERGE, restored.getMergeType()); } @Test - public void testAlterReplayMergesCurrentDefinitionIntoOrigStmt() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "alter_job", 8001L, - 9001L, "127.0.0.1:9092", "alter_topic", UserIdentity.ADMIN); + public void testEmptyDirectStateSafelyFallsBackToOrigStmt() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "empty_job", 8001L, + 9001L, "127.0.0.1:9092", "empty_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.alter_job ON current_table WITH MERGE " - + "COLUMNS TERMINATED BY ',', " - + "COLUMNS(source_col, mapped_col = source_col + 1), " - + "PRECEDING FILTER source_col > 1, WHERE mapped_col < 100, " - + "PARTITION(p1), DELETE ON delete_flag = 1, ORDER BY seq_col " + job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.empty_job ON current_table " + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " - + "\"kafka_topic\" = \"alter_topic\")", 0); + + "\"kafka_topic\" = \"empty_topic\")", 0); - try (MockedStatic ignored = mockCatalog()) { - job = (KafkaRoutineLoadJob) imageRoundTrip(job); - job.replayLoadDefinition(new OriginStatement( - "ALTER ROUTINE LOAD FOR alter_job COLUMNS TERMINATED BY '|', WHERE mapped_col < 50", 0)); - job.replayLoadDefinition(new OriginStatement( - "ALTER ROUTINE LOAD FOR alter_job " - + "PRECEDING FILTER content MATCH_ANY 'hello' USING ANALYZER 'english'", 0)); - } - - Assert.assertTrue(job.origStmt.originStmt.startsWith("CREATE ROUTINE LOAD")); - Assert.assertTrue(job.origStmt.originStmt.contains("COLUMNS TERMINATED BY \"|\"")); - Assert.assertTrue(job.origStmt.originStmt.contains("COLUMNS(")); - Assert.assertTrue(job.origStmt.originStmt.contains("WHERE")); - Assert.assertTrue(job.origStmt.originStmt.contains("PRECEDING FILTER")); - Assert.assertTrue(job.origStmt.originStmt.contains("USING ANALYZER")); - Assert.assertTrue(job.origStmt.originStmt.contains("PARTITION(`p1`)")); - Assert.assertTrue(job.origStmt.originStmt.contains("DELETE ON")); - Assert.assertTrue(job.origStmt.originStmt.contains("ORDER BY `seq_col`")); - Assert.assertTrue(job.origStmt.originStmt.contains("WITH MERGE")); - - JsonObject expectedProperties = JsonParser.parseString(job.jobPropertiesToJsonString()).getAsJsonObject(); RoutineLoadJob restored; try (MockedStatic ignored = mockCatalog()) { restored = imageRoundTrip(job); } - JsonObject restoredProperties = JsonParser.parseString(restored.jobPropertiesToJsonString()).getAsJsonObject(); - for (String key : Lists.newArrayList("column_separator", "precedingFilter", - "whereExpr", "partitions", "delete", "sequence_col", "merge_type")) { - Assert.assertEquals(key, expectedProperties.get(key), restoredProperties.get(key)); - } - Assert.assertTrue(restoredProperties.get("columnToColumnExpr").getAsString().contains("mapped_col=")); - Assert.assertEquals(job.origStmt.originStmt, restored.origStmt.originStmt); + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); + Assert.assertNull(restored.getColumnSeparator()); + Assert.assertNull(restored.getWhereExpr()); + Assert.assertEquals(LoadTask.MergeType.APPEND, restored.getMergeType()); } @Test @@ -155,8 +148,14 @@ public void testLegacyImageContinuesToRestoreFromOrigStmt() throws Exception { JsonObject newImage = imageJson(restored); Assert.assertTrue(newImage.has("ostmt")); - Assert.assertFalse(newImage.has("mt")); - Assert.assertFalse(newImage.has("cs")); + Assert.assertTrue(newImage.has("mt")); + Assert.assertTrue(newImage.has("cs")); + restored.origStmt = new OriginStatement("invalid after legacy recovery", 0); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + RoutineLoadJob restoredAgain = imageRoundTrip(restored); + envStatic.verifyNoInteractions(); + Assert.assertEquals("|", restoredAgain.getColumnSeparator().getSeparator()); + } } private static MockedStatic mockCatalog() throws Exception { @@ -174,6 +173,7 @@ private static MockedStatic mockCatalog() throws Exception { Mockito.when(catalog.getDbOrAnalysisException("legacy_db")).thenReturn(database); Mockito.when(database.getName()).thenReturn("legacy_db"); Mockito.when(database.getFullName()).thenReturn("legacy_db"); + Mockito.when(database.getTable(9001L)).thenReturn(Optional.of((Table) table)); Mockito.when(database.getTableOrMetaException(9001L)).thenReturn(table); Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); Mockito.when(table.getName()).thenReturn("current_table"); @@ -188,6 +188,10 @@ private static MockedStatic mockCatalog() throws Exception { return envStatic; } + private static String exprToSql(Expr expr) { + return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + } + private static JsonObject imageJson(RoutineLoadJob job) throws IOException { return imageJson(writeImage(job)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index afecaedc57eaa6..32770cd8996820 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -17,13 +17,20 @@ package org.apache.doris.persist; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.ImportColumnDesc; +import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.Separator; +import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.load.RoutineLoadDesc; +import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; -import org.apache.doris.qe.OriginStatement; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -58,10 +65,16 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); - OriginStatement originStatement = new OriginStatement( - "ALTER ROUTINE LOAD FOR job WHERE mapped_col > 10", 0); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( + new Separator(",", ","), new Separator("\n", "\\n"), + Lists.newArrayList(new ImportColumnDesc("source_col")), + new BinaryPredicate(BinaryPredicate.Operator.GT, new IntLiteral(2), new IntLiteral(1)), + new BinaryPredicate(BinaryPredicate.Operator.LT, new IntLiteral(1), new IntLiteral(2)), + new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")), + new BinaryPredicate(BinaryPredicate.Operator.EQ, new IntLiteral(1), new IntLiteral(1)), + LoadTask.MergeType.MERGE, "sequence_col"); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, - jobProperties, routineLoadDataSourceProperties, originStatement); + jobProperties, routineLoadDataSourceProperties, routineLoadDesc); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (DataOutputStream out = new DataOutputStream(bytes)) { log.write(out); @@ -82,19 +95,29 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(0)); Assert.assertEquals(routineLoadDataSourceProperties.getKafkaPartitionOffsets().get(1), kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); - Assert.assertEquals(originStatement.originStmt, log2.getOriginStatement().originStmt); - Assert.assertEquals(originStatement.idx, log2.getOriginStatement().idx); + RoutineLoadDesc restoredDesc = log2.getRoutineLoadDesc(); + Assert.assertEquals(",", restoredDesc.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", restoredDesc.getLineDelimiter().getSeparator()); + Assert.assertEquals("source_col", restoredDesc.getColumnsInfo().get(0).getColumnName()); + Assert.assertNotNull(restoredDesc.getPrecedingFilter()); + Assert.assertNotNull(restoredDesc.getFilter()); + Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + restoredDesc.getPartitionNamesInfo().getPartitionNames()); + Assert.assertNotNull(restoredDesc.getDeleteCondition()); + Assert.assertEquals(LoadTask.MergeType.MERGE, restoredDesc.getMergeType()); + Assert.assertEquals("sequence_col", restoredDesc.getSequenceColName()); } @Test - public void testDeserializeLegacyLogWithoutOriginStatement() throws IOException { + public void testDeserializeLegacyLogWithoutRoutineLoadDesc() throws IOException { byte[] bytes = loadBase64Fixture(A8928245_LEGACY_LOG); try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes))) { AlterRoutineLoadJobOperationLog log = AlterRoutineLoadJobOperationLog.read(in); Assert.assertEquals(7001L, log.getJobId()); Assert.assertTrue(log.getJobProperties().isEmpty()); Assert.assertNull(log.getDataSourceProperties()); - Assert.assertNull(log.getOriginStatement()); + Assert.assertNull(log.getRoutineLoadDesc()); } } diff --git a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt index 156e15b3ed20a3..5535e543c20d60 100644 --- a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt +++ b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt @@ -19,4 +19,4 @@ definition. alter-routine-load-log.b64 contains job ID 7001, an empty job-properties map, and a null datasource-properties object. That serializer predates the -originStatement field in AlterRoutineLoadJobOperationLog. +routineLoadDesc field in AlterRoutineLoadJobOperationLog. From 2886172fad709b9d2b5925ab3ebb840a26635ee9 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 15:29:30 +0800 Subject: [PATCH 10/11] [fix](fe) Harden legacy Expr serialization ### What problem does this PR solve? Issue Number: close #66633 Related PR: #66634 Problem Summary: Metadata consumers now persist legacy Expr objects directly, but the existing Expr Gson test only checked subtype and JSON idempotence. Add stable serialization for SQL-relevant fields that were silently dropped, persist function ORDER BY metadata, require every Expr instance field to be serialized or explicitly classified as non-durable, and verify SQL output with and without table names for every concrete registered subtype. Add an analysis review guide for future Expr changes. ### Release note None ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.analysis.ExprGsonSerializationTest - ./run-fe-ut.sh --run org.apache.doris.analysis.ExprGsonSerializationTest,org.apache.doris.load.routineload.RoutineLoadJobPersistenceTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest - Behavior changed: No user-facing SQL behavior; metadata Expr round trips now preserve SQL semantics - Does this need documentation: No, contributor guidance is included in analysis/AGENTS.md --- .../java/org/apache/doris/analysis/AGENTS.md | 26 +++++++ .../doris/analysis/FunctionCallExpr.java | 1 + .../apache/doris/analysis/MatchPredicate.java | 1 + .../apache/doris/analysis/OrderByElement.java | 4 + .../doris/analysis/PlaceHolderExpr.java | 3 + .../doris/analysis/SearchPredicate.java | 3 + .../org/apache/doris/analysis/SlotRef.java | 2 + .../apache/doris/analysis/TimeV2Literal.java | 7 ++ .../apache/doris/analysis/VariableExpr.java | 4 + .../analysis/ExprGsonSerializationTest.java | 74 +++++++++++++++++-- .../RoutineLoadJobPersistenceTest.java | 23 ++++-- 11 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 fe/fe-catalog/src/main/java/org/apache/doris/analysis/AGENTS.md diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/AGENTS.md b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/AGENTS.md new file mode 100644 index 00000000000000..8eb1311c6c0a5f --- /dev/null +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/AGENTS.md @@ -0,0 +1,26 @@ +# Legacy Expr persistence — Review Guide + +Legacy `Expr` objects are persisted by metadata consumers such as Routine Load images and +`ALTER ROUTINE LOAD` journals. Doris Gson serializes only fields annotated with +`@SerializedName`, so an unclassified field can be silently lost during FE recovery. + +## Expr changes + +- [ ] Any field that changes `ExprToSqlVisitor` output has a stable `@SerializedName` key. +- [ ] Existing serialized keys are not renamed, removed, or reused for a different meaning. +- [ ] Analysis caches and execution-only fields remain unpersisted only when they can be rebuilt + after the restored expression is converted to SQL and analyzed again. +- [ ] Every unpersisted instance field is listed with that rationale in + `ExprGsonSerializationTest.NON_DURABLE_EXPR_FIELDS`; do not add fields to the list merely to + make the test pass. +- [ ] New concrete `Expr` subtypes are registered in both Gson factories and have a non-default + sample in `ExprGsonSerializationTest`. +- [ ] Samples set every SQL-relevant option to a non-default value so semantic loss is observable. + +## Required tests + +- [ ] `ExprGsonSerializationTest` preserves the concrete subtype and `ExprToSqlVisitor` output + with and without table names across Gson round-trip. +- [ ] Metadata consumers that introduce a new Expr carrier add an image and journal replay test. +- [ ] Routine Load expression changes cover column mappings, preceding filters, where filters, and + delete conditions as applicable. diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 5a934cd6ca4e26..ec2b4f2bb17424 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -43,6 +43,7 @@ public class FunctionCallExpr extends Expr { private FunctionParams aggFnParams; + @SerializedName("obe") private List orderByElements = Lists.newArrayList(); // check analytic function diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java index d005827ea061cb..23172942d902f2 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java @@ -72,6 +72,7 @@ public String getName() { private String invertedIndexParserStopwords = ""; private String invertedIndexAnalyzerName = ""; // Fields for SQL generation + @SerializedName("ea") private String explicitAnalyzer = ""; private MatchPredicate() { diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/OrderByElement.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/OrderByElement.java index 27b61d59be1094..ea96d0c23cd360 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/OrderByElement.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/OrderByElement.java @@ -21,6 +21,7 @@ package org.apache.doris.analysis; import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; import java.util.List; @@ -28,11 +29,14 @@ * Combination of expr and ASC/DESC, and nulls ordering. */ public class OrderByElement { + @SerializedName("e") private Expr expr; + @SerializedName("ia") private final boolean isAsc; // Represents the NULLs ordering specified: true when "NULLS FIRST", false when // "NULLS LAST", and null if not specified. + @SerializedName("nfp") private final Boolean nullsFirstParam; public OrderByElement(Expr expr, boolean isAsc, Boolean nullsFirstParam) { diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index d4c297196be28b..ea5e3a5e849ac7 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -22,12 +22,15 @@ import org.apache.doris.catalog.Type; import com.google.common.base.Preconditions; +import com.google.gson.annotations.SerializedName; import java.nio.ByteBuffer; // PlaceHolderExpr is a reference class point to real LiteralExpr public class PlaceHolderExpr extends LiteralExpr { + @SerializedName("le") private LiteralExpr lExpr; + @SerializedName("mtc") int mysqlTypeCode = -1; public PlaceHolderExpr() { diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchPredicate.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchPredicate.java index 4c304cf795cfa9..d096fb6f69caa9 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchPredicate.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SearchPredicate.java @@ -21,6 +21,8 @@ import org.apache.doris.catalog.Index; import org.apache.doris.catalog.Type; +import com.google.gson.annotations.SerializedName; + import java.util.Collections; import java.util.List; @@ -29,6 +31,7 @@ * for BE VSearchExpr processing. This is only used during FE->BE translation. */ public class SearchPredicate extends Predicate { + @SerializedName("dsl") private final String dslString; private final QsPlan qsPlan; private final List fieldIndexes; diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java index a2dee823ad386b..6fcea9e05bc82f 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -38,7 +38,9 @@ public class SlotRef extends Expr { @SerializedName("col") private String col; // Used in toSql + @SerializedName("l") private String label; + @SerializedName("scp") private List subColPath; // results of analysis diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java index 96a4014bd59a00..bb225bebbf9b27 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java @@ -20,14 +20,21 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import com.google.gson.annotations.SerializedName; + public class TimeV2Literal extends LiteralExpr { public static final TimeV2Literal MIN_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, true); public static final TimeV2Literal MAX_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, false); + @SerializedName("h") protected int hour; + @SerializedName("m") protected int minute; + @SerializedName("s") protected int second; + @SerializedName("us") protected int microsecond; + @SerializedName("neg") protected boolean negative; /** diff --git a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VariableExpr.java b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VariableExpr.java index 467278c49f23dc..6e3835ce2677d3 100644 --- a/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VariableExpr.java +++ b/fe/fe-catalog/src/main/java/org/apache/doris/analysis/VariableExpr.java @@ -17,13 +17,17 @@ package org.apache.doris.analysis; +import com.google.gson.annotations.SerializedName; + import java.math.BigDecimal; import java.util.Objects; // Variable expr: including the system variable and user define variable. // Converted to StringLiteral in analyze, if this variable is not exist, throw AnalysisException. public class VariableExpr extends Expr { + @SerializedName("n") private String name; + @SerializedName("st") private SetType setType; private boolean isNull; private boolean boolValue; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java index 83b057759ed5a5..bb5bf324ef3a47 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java @@ -51,6 +51,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -59,6 +60,38 @@ public class ExprGsonSerializationTest { private static final Pattern CLASS_DECLARATION_PATTERN = Pattern.compile( "public\\s+(abstract\\s+)?(?:final\\s+)?class\\s+(\\w+)\\s+extends\\s+(\\w+)\\b"); + // These fields are analysis caches or execution-only state. Durable Expr consumers rebuild + // them after converting the restored Expr to SQL and analyzing that SQL again. Keeping the + // classification explicit makes every newly added unannotated Expr field fail this test. + private static final Set NON_DURABLE_EXPR_FIELDS = new TreeSet<>(Arrays.asList( + "BinaryPredicate.slotIsLeft", + "Expr.fn", + "Expr.isConstant", + "Expr.nullable", + "FunctionCallExpr.aggFnParams", + "FunctionCallExpr.isMergeAggFn", + "FunctionCallExpr.originChildSize", + "InformationFunction.intValue", + "InformationFunction.strValue", + "JsonLiteral.beConverted", + "JsonLiteral.parser", + "MatchPredicate.invertedIndexAnalyzerName", + "MatchPredicate.invertedIndexCharFilter", + "MatchPredicate.invertedIndexParser", + "MatchPredicate.invertedIndexParserLowercase", + "MatchPredicate.invertedIndexParserMode", + "MatchPredicate.invertedIndexParserStopwords", + "SearchPredicate.fieldIndexes", + "SearchPredicate.qsPlan", + "SlotRef.desc", + "TryCastExpr.originCastNullable", + "VariableExpr.boolValue", + "VariableExpr.decimalValue", + "VariableExpr.floatValue", + "VariableExpr.intValue", + "VariableExpr.isNull", + "VariableExpr.literalExpr", + "VariableExpr.strValue")); private static class ExprHolder { @SerializedName("expr") @@ -94,7 +127,25 @@ public void testExprRoundTripForAllConcreteRegisteredSubtypes() throws Exception } @Test - public void testExprHolderRoundTrip() { + public void testExprFieldsAreExplicitlyClassifiedForPersistence() throws Exception { + Set unclassifiedFields = new TreeSet<>(); + for (Class exprClass : createExprSamples().keySet()) { + for (Class current = exprClass; Expr.class.isAssignableFrom(current); current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (!field.isSynthetic() && !Modifier.isStatic(modifiers) && !Modifier.isTransient(modifiers) + && field.getAnnotation(SerializedName.class) == null) { + unclassifiedFields.add(current.getSimpleName() + "." + field.getName()); + } + } + } + } + Assertions.assertEquals(NON_DURABLE_EXPR_FIELDS, unclassifiedFields, + "New Expr fields must use @SerializedName or be explicitly classified as non-durable"); + } + + @Test + public void testExprHolderRoundTrip() throws Exception { ExprHolder holder = new ExprHolder( createArithmeticExpr(), Arrays.asList(createSearchPredicate(), createVirtualSlotRef(), createLambdaFunctionExpr())); @@ -111,10 +162,16 @@ public void testExprHolderRoundTrip() { } private void assertExprRoundTrip(Class expectedClass, Expr expr) { + String expectedSqlWithoutTable = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + String expectedSqlWithTable = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class); Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class); Assertions.assertEquals(expectedClass, restored.getClass()); Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored, Expr.class)); + Assertions.assertEquals(expectedSqlWithoutTable, + restored.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE)); + Assertions.assertEquals(expectedSqlWithTable, + restored.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE)); } private Map, Expr> createExprSamples() throws Exception { @@ -164,8 +221,12 @@ private Map, Expr> createExprSamples() throws Exception { } private FunctionCallExpr createFunctionCallExpr() { - return new FunctionCallExpr("ifnull", - Arrays.asList(NullLiteral.create(Type.VARCHAR), new StringLiteral("fallback")), true); + IntLiteral orderKey = new IntLiteral(7L); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("group_concat", + Arrays.asList(new StringLiteral("value"), orderKey), true); + functionCallExpr.setOrderByElements( + Collections.singletonList(new OrderByElement(orderKey, false, null))); + return functionCallExpr; } private LambdaFunctionCallExpr createLambdaFunctionCallExpr() { @@ -224,7 +285,7 @@ private LikePredicate createLikePredicate() { private MatchPredicate createMatchPredicate() { return new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, - new SlotRef(Type.VARCHAR, false), new StringLiteral("hello"), + createNamedSlotRef("content"), new StringLiteral("hello"), Type.BOOLEAN, NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); } @@ -268,13 +329,14 @@ private ArithmeticExpr createArithmeticExpr() { Type.BIGINT, NullableMode.ALWAYS_NOT_NULLABLE, false); } - private SlotRef createSlotRef() { + private SlotRef createSlotRef() throws Exception { SlotRef slotRef = createNamedSlotRef("col1"); slotRef.setType(Type.BIGINT); + setDeclaredField(SlotRef.class, slotRef, "subColPath", Arrays.asList("nested", "leaf")); return slotRef; } - private VirtualSlotRef createVirtualSlotRef() { + private VirtualSlotRef createVirtualSlotRef() throws Exception { String json = GsonUtilsCatalog.GSON.toJson(createSlotRef(), Expr.class) .replace("\"clazz\":\"SlotRef\"", "\"clazz\":\"VirtualSlotRef\""); return (VirtualSlotRef) GsonUtilsCatalog.GSON.fromJson(json, Expr.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 8ee7baf80ecf87..05bca8a946d213 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -22,14 +22,20 @@ import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeV2Literal; import org.apache.doris.analysis.ToSqlParams; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; import org.apache.doris.datasource.CatalogMgr; @@ -67,12 +73,12 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception 9001L, "127.0.0.1:9092", "image_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; job.origStmt = new OriginStatement("deliberately invalid SQL", 0); - Expr columnExpr = new BinaryPredicate( - BinaryPredicate.Operator.GT, new IntLiteral(3), new IntLiteral(2)); - Expr precedingFilter = new BinaryPredicate( - BinaryPredicate.Operator.GE, new IntLiteral(4), new IntLiteral(3)); + Expr columnExpr = new TimeV2Literal(12, 34, 56, 123456, 6, true); + Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, + namedSlot("content"), new StringLiteral("hello world"), Type.BOOLEAN, + NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); Expr whereExpr = new BinaryPredicate( - BinaryPredicate.Operator.LT, new IntLiteral(1), new IntLiteral(2)); + BinaryPredicate.Operator.GT, namedSlot("a`b"), new IntLiteral(10)); Expr deleteCondition = new BinaryPredicate( BinaryPredicate.Operator.EQ, new IntLiteral(1), new IntLiteral(1)); job.setRoutineLoadDesc(new RoutineLoadDesc( @@ -192,6 +198,13 @@ private static String exprToSql(Expr expr) { return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); } + private static SlotRef namedSlot(String column) { + SlotRef slotRef = new SlotRef(null, column); + slotRef.setLabel("`" + column.replace("`", "``") + "`"); + slotRef.setType(Type.VARCHAR); + return slotRef; + } + private static JsonObject imageJson(RoutineLoadJob job) throws IOException { return imageJson(writeImage(job)); } From 6862bb759425c58a6a9ca007d91837f7c9f99ef1 Mon Sep 17 00:00:00 2001 From: Refrain Date: Thu, 20 Aug 2026 16:48:19 +0800 Subject: [PATCH 11/11] [fix](routineload) Restore full direct persistence contract ### What problem does this PR solve? Issue Number: close #66633 Related PR: #66634 Problem Summary: The first restoration of direct Routine Load persistence omitted parts of the previously reviewed design. Restore the exact persistence implementation from commit 4394fa3d4d4, including execMemLimit and memtableOnSinkNode image fields, jobProperties cache hydration, CSV ALTER cache synchronization, leader-only validation, legacy image migration, and the original Kafka/Kinesis persistence tests. Keep the separate Expr serde hardening on top. ### Release note Routine Load persists its effective load definition and non-default task configuration directly across FE recovery. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.analysis.ExprGsonSerializationTest,org.apache.doris.load.routineload.RoutineLoadJobPersistenceTest,org.apache.doris.load.routineload.KafkaRoutineLoadJobTest,org.apache.doris.load.routineload.KinesisRoutineLoadJobTest,org.apache.doris.persist.AlterRoutineLoadOperationLogTest - Behavior changed: Yes, restore the complete direct-state image and ALTER journal persistence contract - Does this need documentation: Yes, document mixed-version ALTER limitations --- .../load/routineload/RoutineLoadJob.java | 252 ++++++++---- .../kafka/KafkaRoutineLoadJob.java | 14 +- .../kinesis/KinesisRoutineLoadJob.java | 14 +- .../routineload/KafkaRoutineLoadJobTest.java | 84 ++-- .../KinesisRoutineLoadJobTest.java | 89 ++--- .../RoutineLoadJobPersistenceTest.java | 370 ++++++++++++++---- .../AlterRoutineLoadOperationLogTest.java | 57 ++- .../routine-load/a8928245/PROVENANCE.txt | 2 +- ...ne_load_alter_checkpoint_restart_fe.groovy | 1 + 9 files changed, 595 insertions(+), 288 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index a30b1cbd2ff109..0973a0e1c76059 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -116,7 +116,6 @@ public abstract class RoutineLoadJob extends AbstractTxnStateChangeCallback implements Writable, LoadTaskInfo, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(RoutineLoadJob.class); - public static final long DEFAULT_MAX_ERROR_NUM = 0; public static final double DEFAULT_MAX_FILTER_RATIO = 1.0; @@ -207,6 +206,7 @@ public boolean isFinalState() { @SerializedName("men") protected long maxErrorNum = DEFAULT_MAX_ERROR_NUM; // optional protected double maxFilterRatio = DEFAULT_MAX_FILTER_RATIO; + @SerializedName("eml") protected long execMemLimit = DEFAULT_EXEC_MEM_LIMIT; protected int sendBatchParallelism = DEFAULT_SEND_BATCH_PARALLELISM; protected boolean loadToSingleTablet = DEFAULT_LOAD_TO_SINGLE_TABLET; @@ -239,6 +239,7 @@ public boolean isFinalState() { @SerializedName("sc") protected String sequenceCol; + @SerializedName("mosn") protected boolean memtableOnSinkNode = false; protected int currentTaskConcurrentNum; @@ -265,7 +266,7 @@ public boolean isFinalState() { // The tasks belong to this job protected List routineLoadTaskInfoList = Lists.newArrayList(); - // Keep the original CREATE statement for downgrade compatibility and legacy image recovery. + // Keep the original CREATE statement for downgrade compatibility and legacy image migration. @SerializedName("ostmt") protected OriginStatement origStmt; // User who submit this job. Maybe null for the old version job(before v1.1) @@ -277,7 +278,7 @@ public boolean isFinalState() { protected ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @SerializedName("mt") - protected LoadTask.MergeType mergeType = LoadTask.MergeType.APPEND; // default is all data is load no delete + protected LoadTask.MergeType mergeType; @SerializedName("dc") protected Expr deleteCondition; // TODO(ml): error sample @@ -324,6 +325,7 @@ public RoutineLoadJob(Long id, String name, this.tableId = tableId; this.authCode = 0; this.userIdentity = userIdentity; + this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -353,6 +355,7 @@ public RoutineLoadJob(Long id, String name, this.authCode = 0; this.userIdentity = userIdentity; this.isMultiTable = true; + this.mergeType = LoadTask.MergeType.APPEND; if (ConnectContext.get() != null) { SessionVariable var = ConnectContext.get().getSessionVariable(); @@ -1975,92 +1978,108 @@ public void gsonPostProcess() throws IOException { if (tableId == 0) { isMultiTable = true; } - // Process UNIQUE_KEY_UPDATE_MODE first to ensure correct backward compatibility - // with PARTIAL_COLUMNS (HashMap iteration order is not guaranteed) - if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { - String modeValue = jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode(modeValue); - if (mode != null) { - uniqueKeyUpdateMode = mode; - isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); - } else { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPSERT; - } - } - // Process remaining properties - jobProperties.forEach((k, v) -> { - if (k.equals(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - // Backward compatibility: only use partial_columns if unique_key_update_mode is not set - // unique_key_update_mode takes precedence - if (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPSERT) { - isPartialUpdate = Boolean.parseBoolean(v); - if (isPartialUpdate) { - uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; - } - } - } else if (k.equals(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - if ("ERROR".equalsIgnoreCase(v)) { - partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } - }); - // Old images have none of the nullable effective load-definition fields. A new image with - // an empty definition also takes this path, which is safe because ALTER cannot unset all - // load clauses and the original CREATE statement therefore has the same empty definition. - if (hasPersistedLoadDefinition()) { - if (userIdentity != null) { - userIdentity.setIsAnalyzed(); - } - return; + // Legacy images did not persist mergeType. New images always contain it, including jobs + // without any load clause, so its absence is sufficient to identify the one-time fallback. + boolean isOldImage = mergeType == null; + if (isOldImage) { + mergeType = LoadTask.MergeType.APPEND; + // Legacy images did not persist this create-time session option. Preserve their historical + // post-restart behavior instead of inheriting the image-loading thread's ConnectContext. + memtableOnSinkNode = false; } try { - ConnectContext ctx = new ConnectContext(); - ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); - StatementContext statementContext = new StatementContext(); - statementContext.setConnectContext(ctx); - ctx.setStatementContext(statementContext); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setCurrentUserIdentity(UserIdentity.ADMIN); - ctx.getState().reset(); - try { - ctx.setThreadLocalInfo(); - NereidsParser nereidsParser = new NereidsParser(); - CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( - origStmt.originStmt); - CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); - // If tableId is set, resolve the current table name by ID so that - // table rename / SWAP TABLE won't cause replay to fail with stale name in origStmt. - if (!isMultiTable && tableId != 0) { - try { - Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); - if (db != null) { - db.getTable(tableId).ifPresent( - table -> createRoutineLoadInfo.setTableName(table.getName())); - } - } catch (Exception ignored) { - // fall through; let validate() surface the real error - } - } - createRoutineLoadInfo.validate(ctx); - setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); - } finally { - ctx.cleanup(); + hydrateJobProperties(); + if (isOldImage) { + restoreLegacyDefinition(); } } catch (Exception e) { this.state = JobState.CANCELLED; - LOG.warn("error happens when parsing create routine load stmt: " + origStmt.originStmt, e); + LOG.warn("error happens when restoring routine load job", e); } if (userIdentity != null) { userIdentity.setIsAnalyzed(); } } - private boolean hasPersistedLoadDefinition() { - return partitionNamesInfo != null || columnDescs != null || precedingFilter != null - || whereExpr != null || columnSeparator != null || lineDelimiter != null - || sequenceCol != null || deleteCondition != null; + private void hydrateJobProperties() throws UserException { + if (jobProperties.containsKey(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)) { + maxFilterRatio = Double.parseDouble( + jobProperties.get(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)) { + sendBatchParallelism = Integer.parseInt( + jobProperties.get(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM)); + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)) { + loadToSingleTablet = Boolean.parseBoolean( + jobProperties.get(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET)); + } + + boolean hasUniqueKeyUpdateMode = jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); + if (hasUniqueKeyUpdateMode) { + TUniqueKeyUpdateMode mode = CreateRoutineLoadInfo.parseUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + uniqueKeyUpdateMode = mode == null ? TUniqueKeyUpdateMode.UPSERT : mode; + isPartialUpdate = uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + if (!hasUniqueKeyUpdateMode && jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { + isPartialUpdate = Boolean.parseBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); + if (isPartialUpdate) { + uniqueKeyUpdateMode = TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS; + } + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase( + jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) + ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; + } + + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { + enclose = parseEnclose(jobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { + escape = parseEscape(jobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + emptyFieldAsNull = Boolean.parseBoolean( + jobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); + } + } + + private void restoreLegacyDefinition() throws UserException { + ConnectContext ctx = new ConnectContext(); + ctx.setDatabase(Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get().getName()); + StatementContext statementContext = new StatementContext(); + statementContext.setConnectContext(ctx); + ctx.setStatementContext(statementContext); + ctx.setEnv(Env.getCurrentEnv()); + ctx.setCurrentUserIdentity(UserIdentity.ADMIN); + ctx.getState().reset(); + try { + ctx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + CreateRoutineLoadCommand command = (CreateRoutineLoadCommand) nereidsParser.parseSingle( + origStmt.originStmt); + CreateRoutineLoadInfo createRoutineLoadInfo = command.getCreateRoutineLoadInfo(); + // Resolve the current table name by ID so table rename or SWAP TABLE does not leave the + // legacy CREATE statement pointing at a stale table name. + if (!isMultiTable && tableId != 0) { + try { + Database db = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).orElse(null); + if (db != null) { + db.getTable(tableId).ifPresent( + table -> createRoutineLoadInfo.setTableName(table.getName())); + } + } catch (Exception ignored) { + // Let validate() below surface the original catalog error. + } + } + createRoutineLoadInfo.validate(ctx); + setRoutineLoadDesc(createRoutineLoadInfo.getRoutineLoadDesc()); + execMemLimit = createRoutineLoadInfo.getExecMemLimit(); + } finally { + ctx.cleanup(); + } } public abstract void modifyProperties(AlterRoutineLoadCommand command) throws UserException; @@ -2069,7 +2088,26 @@ private boolean hasPersistedLoadDefinition() { public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException; - // for ALTER ROUTINE LOAD + // Leader-only validation. Replay must accept values written by older FE versions. + protected void validateCommonJobProperties(Map jobProperties) throws UserException { + validateCsvFormatProperties(jobProperties); + if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { + TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode( + jobProperties.get(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)); + if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { + validateFlexiblePartialUpdateForAlter(); + } + } + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + if (!"APPEND".equalsIgnoreCase(policy) && !"ERROR".equalsIgnoreCase(policy)) { + throw new AnalysisException(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY + + " should be one of {'APPEND', 'ERROR'}, but found " + policy); + } + } + } + + // Apply ALTER ROUTINE LOAD properties. The leader validates before mutation; replay trusts the journal. protected void modifyCommonJobProperties(Map jobProperties) throws UserException { if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) { this.desireTaskConcurrentNum = Integer.parseInt( @@ -2104,12 +2142,7 @@ protected void modifyCommonJobProperties(Map jobProperties) thro if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) { String modeStr = jobProperties.remove(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE); - TUniqueKeyUpdateMode newMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); - // Validate flexible partial update constraints when changing to UPDATE_FLEXIBLE_COLUMNS - if (newMode == TUniqueKeyUpdateMode.UPDATE_FLEXIBLE_COLUMNS) { - validateFlexiblePartialUpdateForAlter(); - } - this.uniqueKeyUpdateMode = newMode; + this.uniqueKeyUpdateMode = CreateRoutineLoadInfo.parseAndValidateUniqueKeyUpdateMode(modeStr); this.isPartialUpdate = (uniqueKeyUpdateMode == TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); @@ -2126,6 +2159,55 @@ protected void modifyCommonJobProperties(Map jobProperties) thro this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); this.jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, uniqueKeyUpdateMode.name()); } + + if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { + String policy = jobProperties.remove(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); + partialUpdateNewKeyPolicy = "ERROR".equalsIgnoreCase(policy) + ? TPartialUpdateNewRowPolicy.ERROR : TPartialUpdateNewRowPolicy.APPEND; + this.jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, + partialUpdateNewKeyPolicy.name()); + } + + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_ENCLOSE); + enclose = parseEnclose(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, value); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_ESCAPE); + escape = parseEscape(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, value); + } + if (jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + String value = jobProperties.remove(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL); + emptyFieldAsNull = Boolean.parseBoolean(value); + this.jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, value); + } + } + + private static void validateCsvFormatProperties(Map jobProperties) { + if (!jobProperties.containsKey(CsvFileFormatProperties.PROP_ENCLOSE) + && !jobProperties.containsKey(CsvFileFormatProperties.PROP_ESCAPE) + && !jobProperties.containsKey(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)) { + return; + } + Map csvProperties = Maps.newHashMap(); + for (String property : new String[] {CsvFileFormatProperties.PROP_ENCLOSE, + CsvFileFormatProperties.PROP_ESCAPE, CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL}) { + if (jobProperties.containsKey(property)) { + csvProperties.put(property, jobProperties.get(property)); + } + } + CsvFileFormatProperties properties = new CsvFileFormatProperties(FileFormatProperties.FORMAT_CSV); + properties.analyzeFileFormatProperties(csvProperties, false); + } + + private static byte parseEnclose(String value) { + return Strings.isNullOrEmpty(value) ? 0 : (byte) value.charAt(0); + } + + private static byte parseEscape(String value) { + return Strings.isNullOrEmpty(value) ? 0 : value.getBytes()[0]; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java index da15d026182147..be124f7c72e6c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java @@ -60,7 +60,6 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -74,7 +73,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -778,6 +776,7 @@ public Map getCustomProperties() { @Override public void modifyProperties(AlterRoutineLoadCommand command) throws UserException { Map jobProperties = command.getAnalyzedJobProperties(); + validateCommonJobProperties(jobProperties); KafkaDataSourceProperties dataSourceProperties = (KafkaDataSourceProperties) command.getDataSourceProperties(); if (null != dataSourceProperties) { // if the partition offset is set by timestamp, convert it to real offset @@ -882,17 +881,6 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if ("ERROR".equalsIgnoreCase(policy)) { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } } LOG.info("modify the properties of kafka routine load job: {}, jobProperties: {}, datasource properties: {}", this.id, jobProperties, dataSourceProperties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java index 8eb62601620acb..ea416e48039df4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java @@ -51,7 +51,6 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFileCompressType; -import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import org.apache.doris.transaction.TransactionState; import org.apache.doris.transaction.TransactionStatus; @@ -65,7 +64,6 @@ import com.google.gson.annotations.SerializedName; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.BooleanUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -686,6 +684,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti throw new DdlException("Only supports modification of PAUSED jobs"); } + validateCommonJobProperties(jobProperties); modifyPropertiesInternal(jobProperties, dataSourceProperties); setRoutineLoadDesc(command.getRoutineLoadDesc()); @@ -763,17 +762,6 @@ private void modifyPropertiesInternal(Map jobProperties, Map copiedJobProperties = Maps.newHashMap(jobProperties); modifyCommonJobProperties(copiedJobProperties); this.jobProperties.putAll(copiedJobProperties); - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_COLUMNS)) { - this.isPartialUpdate = BooleanUtils.toBoolean(jobProperties.get(CreateRoutineLoadInfo.PARTIAL_COLUMNS)); - } - if (jobProperties.containsKey(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY)) { - String policy = jobProperties.get(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY); - if ("ERROR".equalsIgnoreCase(policy)) { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.ERROR; - } else { - this.partialUpdateNewKeyPolicy = TPartialUpdateNewRowPolicy.APPEND; - } - } } LOG.info("modify the properties of kinesis routine load job: {}, jobProperties: {}, datasource properties: {}", this.id, jobProperties, dataSourceProperties); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java index 7e80969d839c25..f8244467502779 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java @@ -30,9 +30,9 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.kafka.KafkaUtil; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kafka.KafkaConfiguration; @@ -49,7 +49,6 @@ import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TResourceInfo; import org.apache.doris.thrift.TRoutineLoadTask; @@ -76,7 +75,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; public class KafkaRoutineLoadJobTest { @@ -104,7 +102,6 @@ public class KafkaRoutineLoadJobTest { @Before public void init() { connectContextStatic = MockedAuth.mockedConnectContext(connectContext, "root", "192.168.1.1"); - Mockito.when(connectContext.getDatabase()).thenReturn("db1"); List partitionNameList = Lists.newArrayList(); partitionNameList.add("p1"); @@ -285,18 +282,19 @@ public void testUpdateProgressWarnsWhenReadCommittedTaskHasZeroRowsAndLag() thro } @Test - public void testAlterPersistsRoutineLoadDescForReplay() throws Exception { + public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exception { KafkaRoutineLoadJob leader = createPausedJob(); KafkaRoutineLoadJob follower = createPausedJob(); RoutineLoadDesc originalDesc = new RoutineLoadDesc(new Separator("|", "|"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, "original_sequence"); leader.setRoutineLoadDesc(originalDesc); follower.setRoutineLoadDesc(originalDesc); - leader.origStmt = initialOriginStatement(); - follower.origStmt = initialOriginStatement(); Map jobProperties = Maps.newHashMap(); - RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, null, + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), null, null, null, null, null, LoadTask.MergeType.APPEND, null); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); @@ -304,28 +302,10 @@ public void testAlterPersistsRoutineLoadDescForReplay() throws Exception { Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); Env env = Mockito.mock(Env.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); EditLog editLog = Mockito.mock(EditLog.class); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(database); - Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("db1"); - Mockito.when(database.getFullName()).thenReturn("db1"); - Mockito.when(database.getTableOrMetaException(1L)).thenReturn(table); - Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("table1"); - Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); AlterRoutineLoadJobOperationLog alterLog; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); Mockito.when(env.getEditLog()).thenReturn(editLog); leader.modifyProperties(command); @@ -334,16 +314,32 @@ public void testAlterPersistsRoutineLoadDescForReplay() throws Exception { ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); alterLog = logCaptor.getValue(); - Assert.assertEquals(";", alterLog.getRoutineLoadDesc().getColumnSeparator().getSeparator()); - Assert.assertEquals(jobProperties, alterLog.getJobProperties()); - assertAlterState(leader); + } - follower.replayModifyProperties(alterLog); - assertAlterState(follower); + Assert.assertSame(delta, alterLog.getRoutineLoadDesc()); + Assert.assertEquals(jobProperties, alterLog.getJobProperties()); + assertAlterState(leader); - assertAlterState(imageRoundTrip(leader)); - assertAlterState(imageRoundTrip(follower)); - } + follower.replayModifyProperties(alterLog); + assertAlterState(follower); + + assertAlterState(imageRoundTrip(leader)); + assertAlterState(imageRoundTrip(follower)); + } + + @Test + public void testReplayLegacyCsvPropertiesDoesNotRunNewValidation() { + KafkaRoutineLoadJob follower = createPausedJob(); + Map legacyJobProperties = Maps.newHashMap(); + legacyJobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "legacy"); + AlterRoutineLoadJobOperationLog legacyLog = new AlterRoutineLoadJobOperationLog( + follower.getId(), legacyJobProperties, null); + + follower.replayModifyProperties(legacyLog); + + Assert.assertEquals((byte) 'l', follower.getEnclose()); + Map persistedJobProperties = Deencapsulation.getField(follower, "jobProperties"); + Assert.assertEquals("legacy", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); } private static KafkaRoutineLoadJob createPausedJob() { @@ -354,16 +350,18 @@ private static KafkaRoutineLoadJob createPausedJob() { } private static void assertAlterState(RoutineLoadJob job) { - Assert.assertEquals(";", job.getColumnSeparator().getSeparator()); - Assert.assertNull(job.getLineDelimiter()); + Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); Assert.assertEquals("original_sequence", job.getSequenceCol()); - } - - private static OriginStatement initialOriginStatement() { - return new OriginStatement("CREATE ROUTINE LOAD db1.job1 ON table1 " - + "COLUMNS TERMINATED BY '|', ORDER BY original_sequence " - + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9020\", " - + "\"kafka_topic\" = \"topic1\")", 0); + Assert.assertEquals((byte) '"', job.getEnclose()); + Assert.assertEquals((byte) '\\', job.getEscape()); + Assert.assertTrue(job.getEmptyFieldAsNull()); + Assert.assertEquals(Boolean.TRUE, Deencapsulation.getField(job, "emptyFieldAsNull")); + + Map persistedJobProperties = Deencapsulation.getField(job, "jobProperties"); + Assert.assertEquals("\"", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE)); + Assert.assertEquals("\\", persistedJobProperties.get(CsvFileFormatProperties.PROP_ESCAPE)); + Assert.assertEquals("true", persistedJobProperties.get(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL)); } private static RoutineLoadJob imageRoundTrip(RoutineLoadJob routineLoadJob) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java index 18fad2af04824b..c59309e60f8aa0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java @@ -19,14 +19,11 @@ import org.apache.doris.analysis.Separator; import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.OlapTable; import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.jmockit.Deencapsulation; -import org.apache.doris.datasource.CatalogMgr; -import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.load.routineload.kinesis.KinesisConfiguration; @@ -34,10 +31,10 @@ import org.apache.doris.load.routineload.kinesis.KinesisProgress; import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; import org.apache.doris.load.routineload.kinesis.KinesisTaskInfo; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.AlterRoutineLoadCommand; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.EditLog; -import org.apache.doris.qe.OriginStatement; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -57,7 +54,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -252,12 +248,15 @@ public void testModifyPropertiesShouldReplaceCustomShardsWhenExplicitShardsProvi } @Test - public void testAlterRoutineLoadDescReplayKeepsCheckpointParity() throws Exception { + public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exception { KinesisRoutineLoadJob leader = createPausedJobWithInitialLoadDesc(); KinesisRoutineLoadJob replay = createPausedJobWithInitialLoadDesc(); Map jobProperties = Maps.newHashMap(); - RoutineLoadDesc delta = new RoutineLoadDesc(new Separator(";", ";"), null, + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + RoutineLoadDesc delta = new RoutineLoadDesc(null, new Separator("\n", "\\n"), null, null, null, null, null, LoadTask.MergeType.APPEND, "sequence_col"); AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); @@ -265,43 +264,41 @@ public void testAlterRoutineLoadDescReplayKeepsCheckpointParity() throws Excepti Mockito.when(command.getRoutineLoadDesc()).thenReturn(delta); Env env = Mockito.mock(Env.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - InternalCatalog catalog = Mockito.mock(InternalCatalog.class); - Database database = Mockito.mock(Database.class); - OlapTable table = Mockito.mock(OlapTable.class); EditLog editLog = Mockito.mock(EditLog.class); - Mockito.when(env.getInternalCatalog()).thenReturn(catalog); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); - Mockito.when(catalog.getDb(1L)).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDb("db1")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrMetaException(1L)).thenReturn(database); - Mockito.when(catalog.getDbOrAnalysisException("db1")).thenReturn(database); - Mockito.when(database.getName()).thenReturn("db1"); - Mockito.when(database.getFullName()).thenReturn("db1"); - Mockito.when(database.getTableOrMetaException(1L)).thenReturn(table); - Mockito.when(database.getTableOrAnalysisException("table1")).thenReturn(table); - Mockito.when(table.getName()).thenReturn("table1"); - Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); ArgumentCaptor logCaptor = ArgumentCaptor.forClass(AlterRoutineLoadJobOperationLog.class); try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); Mockito.when(env.getEditLog()).thenReturn(editLog); leader.modifyProperties(command); Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture()); - AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue()); - replay.replayModifyProperties(log); - - Assert.assertEquals(";", log.getRoutineLoadDesc().getColumnSeparator().getSeparator()); - assertAlterResult(leader); - assertAlterResult(replay); - Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), - JsonParser.parseString(checkpointJson(replay))); - assertAlterResult(imageRoundTrip(leader)); - assertAlterResult(imageRoundTrip(replay)); } + + AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue()); + replay.replayModifyProperties(log); + + Assert.assertNotSame(delta, log.getRoutineLoadDesc()); + Assert.assertEquals("\n", log.getRoutineLoadDesc().getLineDelimiter().getSeparator()); + Assert.assertEquals("sequence_col", log.getRoutineLoadDesc().getSequenceColName()); + assertAlterResult(leader); + assertAlterResult(replay); + Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)), + JsonParser.parseString(checkpointJson(replay))); + } + + @Test + public void testAlterValidatesCsvBeforeDataSourceMutation() { + KinesisRoutineLoadJob job = createPausedJobWithInitialLoadDesc(); + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "invalid"); + KinesisDataSourceProperties dataSourceProperties = Mockito.mock(KinesisDataSourceProperties.class); + AlterRoutineLoadCommand command = Mockito.mock(AlterRoutineLoadCommand.class); + Mockito.when(command.getAnalyzedJobProperties()).thenReturn(jobProperties); + Mockito.when(command.getDataSourceProperties()).thenReturn(dataSourceProperties); + + Assert.assertThrows(AnalysisException.class, () -> job.modifyProperties(command)); + Assert.assertEquals("stream-1", job.getStream()); + Mockito.verifyNoInteractions(dataSourceProperties); } @Test @@ -421,16 +418,16 @@ private KinesisRoutineLoadJob createPausedJobWithInitialLoadDesc() { Deencapsulation.setField(job, "createTimestamp", 123L); job.setRoutineLoadDesc(new RoutineLoadDesc(new Separator("|", "|"), null, null, null, null, null, null, LoadTask.MergeType.APPEND, null)); - job.origStmt = new OriginStatement("CREATE ROUTINE LOAD db1.kinesis_routine_load_job ON table1 " - + "COLUMNS TERMINATED BY '|' FROM KINESIS " - + "(\"aws.region\" = \"us-east-1\", \"kinesis_stream\" = \"stream-1\")", 0); return job; } private void assertAlterResult(KinesisRoutineLoadJob job) { - Assert.assertEquals(";", job.getColumnSeparator().getSeparator()); - Assert.assertNull(job.getLineDelimiter()); + Assert.assertEquals("|", job.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", job.getLineDelimiter().getSeparator()); Assert.assertEquals("sequence_col", job.getSequenceCol()); + Assert.assertEquals((byte) '"', job.getEnclose()); + Assert.assertEquals((byte) '\\', job.getEscape()); + Assert.assertTrue(job.getEmptyFieldAsNull()); } private AlterRoutineLoadJobOperationLog journalRoundTrip(AlterRoutineLoadJobOperationLog log) @@ -454,16 +451,6 @@ private String checkpointJson(RoutineLoadJob job) throws Exception { } } - private KinesisRoutineLoadJob imageRoundTrip(RoutineLoadJob job) throws Exception { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(bytes)) { - job.write(out); - } - try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { - return (KinesisRoutineLoadJob) RoutineLoadJob.read(in); - } - } - private Set collectAssignedShards(KinesisRoutineLoadJob routineLoadJob) { List routineLoadTaskInfoList = Deencapsulation.getField(routineLoadJob, "routineLoadTaskInfoList"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java index 05bca8a946d213..2a8284ad88bec5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java @@ -32,20 +32,29 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Function.NullableMode; -import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.io.Text; +import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.load.RoutineLoadDesc; import org.apache.doris.load.loadv2.LoadTask; +import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob; +import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob; +import org.apache.doris.nereids.load.NereidsRoutineLoadTaskInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.qe.OriginStatement; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; +import org.apache.doris.thrift.TUniqueKeyUpdateMode; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import org.junit.Assert; @@ -61,6 +70,8 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; +import java.util.Map; import java.util.Optional; public class RoutineLoadJobPersistenceTest { @@ -69,32 +80,67 @@ public class RoutineLoadJobPersistenceTest { @Test public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "direct_job", 8001L, - 9001L, "127.0.0.1:9092", "image_topic", UserIdentity.ADMIN); + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1001L, "direct_job", 1002L, + 1003L, "127.0.0.1:9092", "direct_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("deliberately invalid SQL", 0); - Expr columnExpr = new TimeV2Literal(12, 34, 56, 123456, 6, true); + job.origStmt = new OriginStatement("this is deliberately not valid SQL", 0); + + Separator columnSeparator = analyzedSeparator("\\x01"); + Separator lineDelimiter = analyzedSeparator("\\n"); + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true))); + SlotRef matchSlot = namedSlot("content"); Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, - namedSlot("content"), new StringLiteral("hello world"), Type.BOOLEAN, + matchSlot, new StringLiteral("hello world"), Type.BOOLEAN, NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); - Expr whereExpr = new BinaryPredicate( - BinaryPredicate.Operator.GT, namedSlot("a`b"), new IntLiteral(10)); - Expr deleteCondition = new BinaryPredicate( - BinaryPredicate.Operator.EQ, new IntLiteral(1), new IntLiteral(1)); - job.setRoutineLoadDesc(new RoutineLoadDesc( - new Separator("|", "|"), new Separator("\n", "\\n"), - Lists.newArrayList(new ImportColumnDesc("source_col"), - new ImportColumnDesc("mapped_col", columnExpr)), - precedingFilter, whereExpr, - new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")), - deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); + SlotRef quotedSlot = namedSlot("a`b"); + Expr whereExpr = new BinaryPredicate(BinaryPredicate.Operator.GT, quotedSlot, new IntLiteral(10L)); + Expr deleteCondition = predicate(BinaryPredicate.Operator.EQ, "delete_flag", 1L); + PartitionNamesInfo partitions = new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2")); + job.setRoutineLoadDesc(new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, whereExpr, partitions, deleteCondition, LoadTask.MergeType.MERGE, "seq_col")); + String expectedColumnSql = exprToSql(columns.get(1).getExpr()); + String expectedPrecedingSql = exprToSql(precedingFilter); + String expectedWhereSql = exprToSql(whereExpr); + String expectedDeleteSql = exprToSql(deleteCondition); + + job.desireTaskConcurrentNum = 5; + job.maxErrorNum = 17L; + job.maxBatchIntervalS = 23L; + job.maxBatchRows = 300001L; + job.maxBatchSizeBytes = 104857601L; + job.execMemLimit = 345678901L; + job.maxFilterRatio = 0.99; + job.sendBatchParallelism = 99; + job.loadToSingleTablet = false; + job.memtableOnSinkNode = true; + + Map jobProperties = Maps.newHashMap(); + jobProperties.put(CreateRoutineLoadInfo.MAX_FILTER_RATIO_PROPERTY, "0.25"); + jobProperties.put(CreateRoutineLoadInfo.SEND_BATCH_PARALLELISM, "4"); + jobProperties.put(CreateRoutineLoadInfo.LOAD_TO_SINGLE_TABLET, "true"); + jobProperties.put(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE, "UPDATE_FIXED_COLUMNS"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_COLUMNS, "true"); + jobProperties.put(CreateRoutineLoadInfo.PARTIAL_UPDATE_NEW_KEY_POLICY, "ERROR"); + jobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "\""); + jobProperties.put(CsvFileFormatProperties.PROP_ESCAPE, "\\"); + jobProperties.put(CsvFileFormatProperties.PROP_EMPTY_FIELD_AS_NULL, "true"); + job.jobProperties = jobProperties; JsonObject json = imageJson(job); Assert.assertTrue(json.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.MERGE.name(), json.get("mt").getAsString()); for (String key : Lists.newArrayList( - "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc")) { + "pni", "cds", "pf", "we", "cs", "lidel", "sc", "mt", "dc", "eml", "mosn")) { Assert.assertTrue("missing direct-state key " + key, json.has(key)); } + Assert.assertFalse(json.has("ld")); + Assert.assertEquals("\\x01", json.getAsJsonObject("cs").get("os").getAsString()); + Assert.assertEquals("\u0001", json.getAsJsonObject("cs").get("s").getAsString()); + Assert.assertEquals("\\n", json.getAsJsonObject("lidel").get("os").getAsString()); + Assert.assertEquals("\n", json.getAsJsonObject("lidel").get("s").getAsString()); + Assert.assertEquals(2, json.getAsJsonObject("cds").getAsJsonArray("des").size()); RoutineLoadJob restored; try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { @@ -103,68 +149,90 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception } Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals(Lists.newArrayList("p1", "p2"), - restored.getPartitionNamesInfo().getPartitionNames()); - Assert.assertEquals(2, restored.getColumnExprDescs().descs.size()); - Assert.assertEquals(exprToSql(columnExpr), - exprToSql(restored.getColumnExprDescs().descs.get(1).getExpr())); - Assert.assertEquals(exprToSql(precedingFilter), exprToSql(restored.getPrecedingFilter())); - Assert.assertEquals(exprToSql(whereExpr), exprToSql(restored.getWhereExpr())); - Assert.assertEquals(exprToSql(deleteCondition), exprToSql(restored.getDeleteCondition())); - Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), restored.getPartitionNamesInfo().getPartitionNames()); + Assert.assertEquals(2, restored.columnDescs.descs.size()); + Assert.assertEquals("source_col", restored.columnDescs.descs.get(0).getColumnName()); + Assert.assertEquals("mapped_col", restored.columnDescs.descs.get(1).getColumnName()); + Assert.assertNotNull(restored.columnDescs.descs.get(1).getExpr()); + Assert.assertNotNull(restored.getPrecedingFilter()); + Assert.assertNotNull(restored.getWhereExpr()); + Assert.assertEquals(expectedColumnSql, exprToSql(restored.columnDescs.descs.get(1).getExpr())); + Assert.assertEquals(expectedPrecedingSql, exprToSql(restored.getPrecedingFilter())); + Assert.assertEquals(expectedWhereSql, exprToSql(restored.getWhereExpr())); + Assert.assertEquals(expectedDeleteSql, exprToSql(restored.getDeleteCondition())); + Assert.assertEquals("\\x01", restored.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("\u0001", restored.getColumnSeparator().getSeparator()); + Assert.assertEquals("\\n", restored.getLineDelimiter().getOriSeparator()); Assert.assertEquals("\n", restored.getLineDelimiter().getSeparator()); Assert.assertEquals("seq_col", restored.getSequenceCol()); Assert.assertEquals(LoadTask.MergeType.MERGE, restored.getMergeType()); + Assert.assertNotNull(restored.getDeleteCondition()); + Assert.assertEquals(345678901L, restored.getMemLimit()); + Assert.assertTrue(restored.isMemtableOnSinkNode()); + Assert.assertEquals(5, restored.desireTaskConcurrentNum); + Assert.assertEquals(17L, restored.maxErrorNum); + Assert.assertEquals(23L, restored.getMaxBatchIntervalS()); + Assert.assertEquals(300001L, restored.getMaxBatchRows()); + Assert.assertEquals(104857601L, restored.getMaxBatchSizeBytes()); + + NereidsRoutineLoadTaskInfo taskInfo = restored.toNereidsRoutineLoadTaskInfo(); + Assert.assertEquals(345678901L, taskInfo.getMemLimit()); + Assert.assertEquals(0.25, taskInfo.getMaxFilterRatio(), 0.0); + Assert.assertEquals(4, taskInfo.getSendBatchParallelism()); + Assert.assertTrue(taskInfo.isLoadToSingleTablet()); + Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, taskInfo.getUniqueKeyUpdateMode()); + Assert.assertTrue(taskInfo.isFixedPartialUpdate()); + Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, taskInfo.getPartialUpdateNewRowPolicy()); + Assert.assertEquals((byte) '"', taskInfo.getEnclose()); + Assert.assertEquals((byte) '\\', taskInfo.getEscape()); + Assert.assertTrue(taskInfo.getEmptyFieldAsNull()); + Assert.assertTrue(taskInfo.isMemtableOnSinkNode()); + Assert.assertEquals(LoadTask.MergeType.MERGE, taskInfo.getMergeType()); + Assert.assertNotNull(taskInfo.getDeleteCondition()); + Assert.assertEquals("seq_col", taskInfo.getSequenceCol()); + Assert.assertEquals(Lists.newArrayList("p1", "p2"), + taskInfo.getPartitionNamesInfo().getPartitionNames()); + Assert.assertEquals(2, taskInfo.getColumnExprDescs().descs.size()); + Assert.assertNotNull(taskInfo.getPrecedingFilter()); + Assert.assertNotNull(taskInfo.getWhereExpr()); + Assert.assertEquals("\u0001", taskInfo.getColumnSeparator().getSeparator()); + Assert.assertEquals("\n", taskInfo.getLineDelimiter().getSeparator()); } @Test - public void testEmptyDirectStateSafelyFallsBackToOrigStmt() throws Exception { - KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "empty_job", 8001L, - 9001L, "127.0.0.1:9092", "empty_topic", UserIdentity.ADMIN); + public void testDirectStateImageWithNoLoadClausesDoesNotFallback() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(2001L, "empty_job", 2002L, + 2003L, "127.0.0.1:9092", "empty_topic", UserIdentity.ADMIN); job.state = RoutineLoadJob.JobState.PAUSED; - job.origStmt = new OriginStatement("CREATE ROUTINE LOAD legacy_db.empty_job ON current_table " - + "FROM KAFKA (\"kafka_broker_list\" = \"127.0.0.1:9092\", " - + "\"kafka_topic\" = \"empty_topic\")", 0); + job.origStmt = new OriginStatement("also not valid SQL", 0); + + JsonObject json = imageJson(job); + Assert.assertTrue(json.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.APPEND.name(), json.get("mt").getAsString()); + for (String key : Lists.newArrayList("pni", "cds", "pf", "we", "cs", "lidel", "sc", "dc")) { + Assert.assertFalse("unexpected nullable direct-state key " + key, json.has(key)); + } RoutineLoadJob restored; - try (MockedStatic ignored = mockCatalog()) { + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { restored = imageRoundTrip(job); + envStatic.verifyNoInteractions(); } Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertNull(restored.getColumnSeparator()); + Assert.assertNull(restored.getPartitionNamesInfo()); + Assert.assertNull(restored.columnDescs); + Assert.assertNull(restored.getPrecedingFilter()); Assert.assertNull(restored.getWhereExpr()); + Assert.assertNull(restored.getColumnSeparator()); + Assert.assertNull(restored.getLineDelimiter()); + Assert.assertNull(restored.getSequenceCol()); + Assert.assertNull(restored.getDeleteCondition()); Assert.assertEquals(LoadTask.MergeType.APPEND, restored.getMergeType()); } @Test - public void testLegacyImageContinuesToRestoreFromOrigStmt() throws Exception { - byte[] legacyImage = loadBase64Fixture(LEGACY_IMAGE); - JsonObject legacyJson = imageJson(legacyImage); - Assert.assertTrue(legacyJson.has("ostmt")); - Assert.assertFalse(legacyJson.has("mt")); - - RoutineLoadJob restored; - try (MockedStatic ignored = mockCatalog()) { - restored = readImage(legacyImage); - } - - Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restored.getState()); - Assert.assertEquals("|", restored.getColumnSeparator().getSeparator()); - - JsonObject newImage = imageJson(restored); - Assert.assertTrue(newImage.has("ostmt")); - Assert.assertTrue(newImage.has("mt")); - Assert.assertTrue(newImage.has("cs")); - restored.origStmt = new OriginStatement("invalid after legacy recovery", 0); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - RoutineLoadJob restoredAgain = imageRoundTrip(restored); - envStatic.verifyNoInteractions(); - Assert.assertEquals("|", restoredAgain.getColumnSeparator().getSeparator()); - } - } - - private static MockedStatic mockCatalog() throws Exception { + public void testLegacyImageMigratesOnce() throws Exception { Env env = Mockito.mock(Env.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); InternalCatalog catalog = Mockito.mock(InternalCatalog.class); @@ -175,27 +243,171 @@ private static MockedStatic mockCatalog() throws Exception { Mockito.when(catalogMgr.getCatalog(Mockito.anyString())).thenReturn(catalog); Mockito.when(catalog.getDb(8001L)).thenReturn(Optional.of(database)); Mockito.when(catalog.getDb("legacy_db")).thenReturn(Optional.of(database)); - Mockito.when(catalog.getDbOrMetaException(8001L)).thenReturn(database); Mockito.when(catalog.getDbOrAnalysisException("legacy_db")).thenReturn(database); Mockito.when(database.getName()).thenReturn("legacy_db"); - Mockito.when(database.getFullName()).thenReturn("legacy_db"); Mockito.when(database.getTable(9001L)).thenReturn(Optional.of((Table) table)); - Mockito.when(database.getTableOrMetaException(9001L)).thenReturn(table); Mockito.when(database.getTableOrAnalysisException("current_table")).thenReturn(table); Mockito.when(table.getName()).thenReturn("current_table"); Mockito.when(table.getType()).thenReturn(Table.TableType.OLAP); - Mockito.when(table.getKeysType()).thenReturn(KeysType.UNIQUE_KEYS); - Mockito.when(table.hasDeleteSign()).thenReturn(true); - Mockito.when(table.getFullSchema()).thenReturn(Lists.newArrayList()); + Mockito.when(table.getEnableUniqueKeyMergeOnWrite()).thenReturn(true); + + byte[] legacyImage = loadBase64Fixture(LEGACY_IMAGE); + JsonObject legacyJson = imageJson(legacyImage); + Assert.assertFalse(legacyJson.has("mt")); + Assert.assertTrue(legacyJson.has("ostmt")); + + RoutineLoadJob migrated; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + migrated = readImage(legacyImage); + } + + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, migrated.getState()); + Assert.assertEquals("|", migrated.getColumnSeparator().getOriSeparator()); + Assert.assertEquals("|", migrated.getColumnSeparator().getSeparator()); + Assert.assertNull(migrated.getSequenceCol()); + Assert.assertEquals(33554432L, migrated.getMemLimit()); + Assert.assertEquals(0.25, migrated.getMaxFilterRatio(), 0.0); + Assert.assertEquals(3, migrated.getSendBatchParallelism()); + Assert.assertTrue(migrated.isLoadToSingleTablet()); + Assert.assertEquals(TUniqueKeyUpdateMode.UPDATE_FIXED_COLUMNS, migrated.getUniqueKeyUpdateMode()); + Assert.assertTrue(migrated.isFixedPartialUpdate()); + Assert.assertEquals(TPartialUpdateNewRowPolicy.ERROR, migrated.partialUpdateNewKeyPolicy); + Assert.assertEquals((byte) '"', migrated.getEnclose()); + Assert.assertEquals((byte) '\\', migrated.getEscape()); + Assert.assertTrue(migrated.getEmptyFieldAsNull()); + Assert.assertFalse(migrated.isMemtableOnSinkNode()); - MockedStatic envStatic = Mockito.mockStatic(Env.class); - envStatic.when(Env::getCurrentEnv).thenReturn(env); - envStatic.when(Env::getCurrentInternalCatalog).thenReturn(catalog); - return envStatic; + JsonObject migratedJson = imageJson(migrated); + Assert.assertTrue(migratedJson.has("ostmt")); + Assert.assertEquals(LoadTask.MergeType.APPEND.name(), migratedJson.get("mt").getAsString()); + Assert.assertTrue(migratedJson.has("cs")); + Assert.assertTrue(migratedJson.has("eml")); + Assert.assertTrue(migratedJson.has("mosn")); + migrated.origStmt = new OriginStatement("invalid after successful migration", 0); + + RoutineLoadJob restoredAgain; + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + restoredAgain = imageRoundTrip(migrated); + envStatic.verifyNoInteractions(); + } + Assert.assertEquals(RoutineLoadJob.JobState.PAUSED, restoredAgain.getState()); + Assert.assertEquals("|", restoredAgain.getColumnSeparator().getSeparator()); + Assert.assertEquals(33554432L, restoredAgain.getMemLimit()); + Assert.assertFalse(restoredAgain.isMemtableOnSinkNode()); } - private static String exprToSql(Expr expr) { - return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + @Test + public void testKafkaDerivedStateIsRebuiltFromDurableProperties() throws Exception { + KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(3001L, "kafka_derived", 3002L, + 3003L, "127.0.0.1:9092", "derived_topic", UserIdentity.ADMIN); + job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); + Map customProperties = Maps.newHashMap(); + customProperties.put("client.id", "durable-client"); + customProperties.put(KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName(), "OFFSET_BEGINNING"); + Deencapsulation.setField(job, "customProperties", customProperties); + Deencapsulation.setField(job, "customKafkaPartitions", Lists.newArrayList(9)); + Deencapsulation.setField(job, "currentKafkaPartitions", Lists.newArrayList(1, 2)); + Deencapsulation.setField(job, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("stale", "value"))); + Deencapsulation.setField(job, "cachedPartitionWithLatestOffsets", + Maps.newHashMap(ImmutableMap.of(1, 100L))); + Deencapsulation.setField(job, "newCurrentKafkaPartition", Lists.newArrayList(3)); + Deencapsulation.setField(job, "kafkaDefaultOffSet", "OFFSET_END"); + + JsonObject json = imageJson(job); + Assert.assertEquals("127.0.0.1:9092", json.get("bl").getAsString()); + Assert.assertEquals("derived_topic", json.get("tp").getAsString()); + Assert.assertEquals("durable-client", json.getAsJsonObject("prop").get("client.id").getAsString()); + Assert.assertEquals(1, json.getAsJsonArray("cskp").size()); + assertNoJavaFieldNames(json, "currentKafkaPartitions", "convertedCustomProperties", + "cachedPartitionWithLatestOffsets", "newCurrentKafkaPartition", "kafkaDefaultOffSet"); + + KafkaRoutineLoadJob restored = (KafkaRoutineLoadJob) imageRoundTrip(job); + Assert.assertEquals("127.0.0.1:9092", restored.getBrokerList()); + Assert.assertEquals("derived_topic", restored.getTopic()); + Assert.assertEquals(Lists.newArrayList(9), Deencapsulation.getField(restored, "customKafkaPartitions")); + Assert.assertTrue(((List) Deencapsulation.getField(restored, "currentKafkaPartitions")).isEmpty()); + Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); + Assert.assertTrue(((Map) Deencapsulation.getField( + restored, "cachedPartitionWithLatestOffsets")).isEmpty()); + Assert.assertEquals("", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + + Env env = Mockito.mock(Env.class); + try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { + envStatic.when(Env::getCurrentEnv).thenReturn(env); + restored.prepare(); + } + Assert.assertEquals("durable-client", restored.getConvertedCustomProperties().get("client.id")); + Assert.assertFalse(restored.getConvertedCustomProperties().containsKey( + KafkaConfiguration.KAFKA_ORIGIN_DEFAULT_OFFSETS.getName())); + Assert.assertEquals("OFFSET_BEGINNING", Deencapsulation.getField(restored, "kafkaDefaultOffSet")); + } + + @Test + public void testKinesisDerivedStateIsRebuiltFromDurableProperties() throws Exception { + KinesisRoutineLoadJob job = new KinesisRoutineLoadJob(4001L, "kinesis_derived", 4002L, + 4003L, "us-east-1", "derived_stream", UserIdentity.ADMIN); + job.origStmt = new OriginStatement("invalid SQL must stay unused", 0); + Deencapsulation.setField(job, "endpoint", "https://kinesis.example.test"); + Map customProperties = Maps.newHashMap(); + customProperties.put("client.setting", "durable-value"); + customProperties.put("kinesis_default_pos", "TRIM_HORIZON"); + Deencapsulation.setField(job, "customProperties", customProperties); + Deencapsulation.setField(job, "customKinesisShards", Lists.newArrayList("custom-shard")); + Deencapsulation.setField(job, "openKinesisShards", Lists.newArrayList("open-shard")); + Deencapsulation.setField(job, "closedKinesisShards", Lists.newArrayList("closed-shard")); + Deencapsulation.setField(job, "convertedCustomProperties", + Maps.newHashMap(ImmutableMap.of("stale", "value"))); + Deencapsulation.setField(job, "cachedShardWithMillsBehindLatest", + Maps.newHashMap(ImmutableMap.of("open-shard", 99L))); + Deencapsulation.setField(job, "newCurrentKinesisShards", Lists.newArrayList("new-shard")); + Deencapsulation.setField(job, "kinesisDefaultPosition", "LATEST"); + + JsonObject json = imageJson(job); + Assert.assertEquals("us-east-1", json.get("rg").getAsString()); + Assert.assertEquals("derived_stream", json.get("stm").getAsString()); + Assert.assertEquals("https://kinesis.example.test", json.get("ep").getAsString()); + Assert.assertEquals("durable-value", + json.getAsJsonObject("prop").get("client.setting").getAsString()); + Assert.assertEquals("custom-shard", json.getAsJsonArray("csks").get(0).getAsString()); + Assert.assertEquals("open-shard", json.getAsJsonArray("opks").get(0).getAsString()); + Assert.assertEquals("closed-shard", json.getAsJsonArray("clks").get(0).getAsString()); + assertNoJavaFieldNames(json, "convertedCustomProperties", "cachedShardWithMillsBehindLatest", + "newCurrentKinesisShards", "kinesisDefaultPosition"); + + KinesisRoutineLoadJob restored = (KinesisRoutineLoadJob) imageRoundTrip(job); + Assert.assertEquals("us-east-1", restored.getRegion()); + Assert.assertEquals("derived_stream", restored.getStream()); + Assert.assertEquals("https://kinesis.example.test", restored.getEndpoint()); + Assert.assertEquals(Lists.newArrayList("custom-shard"), + Deencapsulation.getField(restored, "customKinesisShards")); + Assert.assertEquals(Lists.newArrayList("open-shard"), + Deencapsulation.getField(restored, "openKinesisShards")); + Assert.assertEquals(Lists.newArrayList("closed-shard"), + Deencapsulation.getField(restored, "closedKinesisShards")); + Assert.assertTrue(restored.getConvertedCustomProperties().isEmpty()); + Assert.assertTrue(((Map) Deencapsulation.getField( + restored, "cachedShardWithMillsBehindLatest")).isEmpty()); + Assert.assertTrue(((List) Deencapsulation.getField(restored, "newCurrentKinesisShards")).isEmpty()); + Assert.assertEquals("", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + + restored.prepare(); + Assert.assertEquals("durable-value", restored.getConvertedCustomProperties().get("client.setting")); + Assert.assertEquals("TRIM_HORIZON", + restored.getConvertedCustomProperties().get("kinesis_default_pos")); + Assert.assertEquals("TRIM_HORIZON", Deencapsulation.getField(restored, "kinesisDefaultPosition")); + } + + private static Separator analyzedSeparator(String value) throws Exception { + Separator separator = new Separator(value); + separator.analyze(); + return separator; + } + + private static Expr predicate(BinaryPredicate.Operator operator, String column, long value) { + return new BinaryPredicate(operator, new SlotRef(null, column), new IntLiteral(value)); } private static SlotRef namedSlot(String column) { @@ -205,6 +417,10 @@ private static SlotRef namedSlot(String column) { return slotRef; } + private static String exprToSql(Expr expr) { + return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + } + private static JsonObject imageJson(RoutineLoadJob job) throws IOException { return imageJson(writeImage(job)); } @@ -242,4 +458,10 @@ private static byte[] loadBase64Fixture(String resource) throws IOException { return Base64.getDecoder().decode(base64); } } + + private static void assertNoJavaFieldNames(JsonObject json, String... fieldNames) { + for (String fieldName : fieldNames) { + Assert.assertFalse("derived field leaked into image: " + fieldName, json.has(fieldName)); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java index 32770cd8996820..7fab0f25108f8c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java @@ -18,9 +18,18 @@ package org.apache.doris.persist; import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprToSqlVisitor; import org.apache.doris.analysis.ImportColumnDesc; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.analysis.Separator; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.analysis.TimeV2Literal; +import org.apache.doris.analysis.ToSqlParams; +import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.info.PartitionNamesInfo; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; @@ -29,6 +38,7 @@ import org.apache.doris.load.routineload.kafka.KafkaConfiguration; import org.apache.doris.load.routineload.kafka.KafkaDataSourceProperties; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -43,6 +53,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; import java.util.Map; public class AlterRoutineLoadOperationLogTest { @@ -65,14 +76,24 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User routineLoadDataSourceProperties.setTimezone(TimeUtils.DEFAULT_TIME_ZONE); routineLoadDataSourceProperties.analyze(); - RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc( - new Separator(",", ","), new Separator("\n", "\\n"), - Lists.newArrayList(new ImportColumnDesc("source_col")), - new BinaryPredicate(BinaryPredicate.Operator.GT, new IntLiteral(2), new IntLiteral(1)), - new BinaryPredicate(BinaryPredicate.Operator.LT, new IntLiteral(1), new IntLiteral(2)), - new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")), - new BinaryPredicate(BinaryPredicate.Operator.EQ, new IntLiteral(1), new IntLiteral(1)), - LoadTask.MergeType.MERGE, "sequence_col"); + Separator columnSeparator = new Separator(",", "\\x2c"); + Separator lineDelimiter = new Separator("\n", "\\n"); + List columns = Lists.newArrayList( + new ImportColumnDesc("source_col"), + new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true))); + Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY, + namedSlot("content"), new StringLiteral("hello world"), Type.BOOLEAN, + NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"); + Expr where = new BinaryPredicate(BinaryPredicate.Operator.GT, + namedSlot("a`b"), new IntLiteral(10L)); + PartitionNamesInfo partitions = new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2")); + BinaryPredicate deleteCondition = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new IntLiteral(1L), new IntLiteral(1L)); + RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns, + precedingFilter, where, partitions, deleteCondition, LoadTask.MergeType.MERGE, "sequence_col"); + String expectedColumnSql = exprToSql(columns.get(1).getExpr()); + String expectedPrecedingSql = exprToSql(precedingFilter); + String expectedWhereSql = exprToSql(where); AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId, jobProperties, routineLoadDataSourceProperties, routineLoadDesc); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); @@ -97,16 +118,25 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User kafkaDataSourceProperties.getKafkaPartitionOffsets().get(1)); RoutineLoadDesc restoredDesc = log2.getRoutineLoadDesc(); Assert.assertEquals(",", restoredDesc.getColumnSeparator().getSeparator()); + Assert.assertEquals("\\x2c", restoredDesc.getColumnSeparator().getOriSeparator()); Assert.assertEquals("\n", restoredDesc.getLineDelimiter().getSeparator()); + Assert.assertEquals("\\n", restoredDesc.getLineDelimiter().getOriSeparator()); + Assert.assertEquals(2, restoredDesc.getColumnsInfo().size()); Assert.assertEquals("source_col", restoredDesc.getColumnsInfo().get(0).getColumnName()); + Assert.assertEquals("mapped_col", restoredDesc.getColumnsInfo().get(1).getColumnName()); + Assert.assertNotNull(restoredDesc.getColumnsInfo().get(1).getExpr()); Assert.assertNotNull(restoredDesc.getPrecedingFilter()); Assert.assertNotNull(restoredDesc.getFilter()); + Assert.assertEquals(expectedColumnSql, exprToSql(restoredDesc.getColumnsInfo().get(1).getExpr())); + Assert.assertEquals(expectedPrecedingSql, exprToSql(restoredDesc.getPrecedingFilter())); + Assert.assertEquals(expectedWhereSql, exprToSql(restoredDesc.getFilter())); Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp()); Assert.assertEquals(Lists.newArrayList("p1", "p2"), restoredDesc.getPartitionNamesInfo().getPartitionNames()); Assert.assertNotNull(restoredDesc.getDeleteCondition()); Assert.assertEquals(LoadTask.MergeType.MERGE, restoredDesc.getMergeType()); Assert.assertEquals("sequence_col", restoredDesc.getSequenceColName()); + Assert.assertEquals(GsonUtils.GSON.toJson(routineLoadDesc), GsonUtils.GSON.toJson(restoredDesc)); } @Test @@ -131,4 +161,15 @@ private static byte[] loadBase64Fixture(String resource) throws IOException { } } + private static SlotRef namedSlot(String column) { + SlotRef slotRef = new SlotRef(null, column); + slotRef.setLabel("`" + column.replace("`", "``") + "`"); + slotRef.setType(Type.VARCHAR); + return slotRef; + } + + private static String exprToSql(Expr expr) { + return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE); + } + } diff --git a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt index 5535e543c20d60..0d24f78c090320 100644 --- a/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt +++ b/fe/fe-core/src/test/resources/upgrade/routine-load/a8928245/PROVENANCE.txt @@ -19,4 +19,4 @@ definition. alter-routine-load-log.b64 contains job ID 7001, an empty job-properties map, and a null datasource-properties object. That serializer predates the -routineLoadDesc field in AlterRoutineLoadJobOperationLog. +RoutineLoadDesc field in AlterRoutineLoadJobOperationLog. diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy index 0045c740660eca..a40a47993608e4 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter_checkpoint_restart_fe.groovy @@ -45,6 +45,7 @@ suite("test_routine_load_alter_checkpoint_restart_fe", "docker") { "column_separator", "precedingFilter", "whereExpr", + "exec_mem_limit", "merge_type" ]