From e2de05ed79612c17f8371def94ecbf810fa25c33 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Fri, 18 Sep 2026 17:38:05 +0800 Subject: [PATCH] [fix](fe) Reject non-sortable duplicate keys when adding a rollup ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: `ALTER TABLE ... ADD ROLLUP r (v, id) DUPLICATE KEY(v)` is accepted by FE when `v` is a VARIANT column, but the asynchronous rollup job then fails in BE and ends up CANCELLED: ``` rollup tasks failed on same tablet reach threshold 1, reason=task type: ALTER, status_code: NOT_IMPLEMENTED_ERROR, status_message: [NOT_IMPLEMENTED_ERROR] not implemented ``` Root cause: when the user lists the duplicate keys explicitly, `MaterializedViewHandler.checkAndPrepareMaterializedView(AddRollupOp ...)` never checks the key column type. The implicit branch stops at the first column that `couldBeShortKey()` rejects, and CREATE MATERIALIZED VIEW rejects these types too, but the explicit branch lets them through. BE sorts the rollup rows by the duplicate keys and cannot compare these types. Checked on a cluster before the fix: | duplicate key type | rollup job result | |---|---| | VARIANT | CANCELLED, NOT_IMPLEMENTED_ERROR | | ARRAY / MAP / STRUCT | CANCELLED, `OlapColumnDataConvertor* not support get_data_at` | | JSON | CANCELLED, INTERNAL_ERROR `value 65537 cast to type t out of range` | | STRING / DOUBLE / IPV6 | FINISHED | Fix: reject a duplicate key whose type `isOnlyMetricType()` (hll, bitmap, quantile_state, array, map, struct, jsonb, variant) when the statement is submitted, with the same `Type.OnlyMetricTypeErrorMsg` that CREATE MATERIALIZED VIEW uses. Types that BE can sort, such as STRING and DOUBLE, keep working as before. The same function also serves the ROLLUP clause of CREATE TABLE, which now reports this reason too. ### Release note ADD ROLLUP with an explicit DUPLICATE KEY on a VARIANT, ARRAY, MAP, STRUCT, JSON, HLL, BITMAP or QUANTILE_STATE column is now rejected immediately instead of creating a job that fails later. ### Check List (For Author) - Test: Regression test (rollup_p0/test_rollup_dup_key_type) - Behavior changed: Yes. These rollups are rejected when submitted instead of being cancelled asynchronously. - Does this need documentation: No Co-Authored-By: Claude Opus 5 --- .../doris/alter/MaterializedViewHandler.java | 7 ++ .../rollup_p0/test_rollup_dup_key_type.out | 5 ++ .../rollup_p0/test_rollup_dup_key_type.groovy | 70 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 regression-test/data/rollup_p0/test_rollup_dup_key_type.out create mode 100644 regression-test/suites/rollup_p0/test_rollup_dup_key_type.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 77fb98b3d70ec7..a197660f284c28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -40,6 +40,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -904,6 +905,12 @@ public List checkAndPrepareMaterializedView(AddRollupOp addRollupOp, Ola throw new DdlException("Invalid column order. key should before all values: " + rollupColName); } + // BE sorts rollup rows by the duplicate keys, which these types cannot be compared on. + if (isKey && baseColumn.getType().isOnlyMetricType()) { + throw new DdlException("Column[" + rollupColName + "] can not be used as a duplicate key " + + "of rollup. " + Type.OnlyMetricTypeErrorMsg); + } + Column oneColumn = new Column(baseColumn); if (isKey) { hasKey = true; diff --git a/regression-test/data/rollup_p0/test_rollup_dup_key_type.out b/regression-test/data/rollup_p0/test_rollup_dup_key_type.out new file mode 100644 index 00000000000000..8d6a54da2b1b2f --- /dev/null +++ b/regression-test/data/rollup_p0/test_rollup_dup_key_type.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !rollup_s -- +x 1 10 +y 2 20 + diff --git a/regression-test/suites/rollup_p0/test_rollup_dup_key_type.groovy b/regression-test/suites/rollup_p0/test_rollup_dup_key_type.groovy new file mode 100644 index 00000000000000..e21cf756cfc6ef --- /dev/null +++ b/regression-test/suites/rollup_p0/test_rollup_dup_key_type.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Explicit rollup duplicate keys must be sortable: FE rejects them up front instead of +// letting the asynchronous rollup job fail in BE. +suite("test_rollup_dup_key_type") { + sql "DROP TABLE IF EXISTS test_rollup_dup_key_type" + sql """ + CREATE TABLE test_rollup_dup_key_type ( + id INT, + v VARIANT, + a ARRAY, + m MAP, + st STRUCT, + j JSON, + s STRING + ) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """INSERT INTO test_rollup_dup_key_type SELECT 1, parse_to_variant('{"a": 10}'), [1], map(1, 1), + named_struct('x', 1), '{"a": 1}', 'x'""" + sql """INSERT INTO test_rollup_dup_key_type SELECT 2, parse_to_variant('{"a": 20}'), [2], map(2, 2), + named_struct('x', 2), '{"a": 2}', 'y'""" + + for (String col : ["v", "a", "m", "st", "j"]) { + test { + sql "ALTER TABLE test_rollup_dup_key_type ADD ROLLUP r_${col} (${col}, id) DUPLICATE KEY(${col})" + exception "Column[${col}] can not be used as a duplicate key of rollup" + } + } + + // A value column of the same types is still allowed after a sortable key. + sql "ALTER TABLE test_rollup_dup_key_type ADD ROLLUP r_s (s, id, v) DUPLICATE KEY(s)" + def state = "" + for (int i = 0; i < 120; i++) { + state = sql("SHOW ALTER TABLE ROLLUP WHERE TableName = 'test_rollup_dup_key_type' " + + "ORDER BY CreateTime DESC LIMIT 1")[0][8] + if (state == "FINISHED" || state == "CANCELLED") { + break + } + sleep(1000) + } + assertEquals("FINISHED", state) + order_qt_rollup_s "SELECT s, id, CAST(v['a'] AS INT) FROM test_rollup_dup_key_type" + + sql "DROP TABLE IF EXISTS test_rollup_dup_key_type_create" + test { + sql """ + CREATE TABLE test_rollup_dup_key_type_create (id INT, v VARIANT) + DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 + ROLLUP (r_v (v, id) DUPLICATE KEY(v)) + PROPERTIES ("replication_num" = "1") + """ + exception "Column[v] can not be used as a duplicate key of rollup" + } +}