From 8e03a6bc0d010babd01832d222dcacb9a4e2408c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 24 Apr 2026 12:38:23 +0800 Subject: [PATCH 1/2] Fixed the schema quota that counts audit DB (#17548) (cherry picked from commit 1e723bcbacf352c0360d27125e08f56d56b0550f) --- .../iotdb/db/schemaengine/SchemaEngine.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java index c8faf2f059a42..4784a168768dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; @@ -393,7 +394,10 @@ public Map countTimeSeriesNumBySchemaRegion(final List s .filter( entry -> targetSchemaIds.contains(entry.getKey().getId()) - && SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey())) + && SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey()) + // Audit logs are stored under the internal system database in dev/1.3. + && !SchemaConstant.SYSTEM_DATABASE.equals( + entry.getValue().getDatabaseFullPath())) .forEach( entry -> timeSeriesNum.put( @@ -442,7 +446,15 @@ public void updateAndFillSchemaCountMap(TDataNodeHeartbeatReq req, TDataNodeHear SchemaRegionConsensusImpl.getInstance().getAllConsensusGroupIds().stream() .filter( consensusGroupId -> - SchemaRegionConsensusImpl.getInstance().isLeader(consensusGroupId)) + SchemaRegionConsensusImpl.getInstance().isLeader(consensusGroupId) + && Optional.ofNullable(schemaRegionMap.get((SchemaRegionId) consensusGroupId)) + .map( + schemaRegion -> + // Audit logs are stored under the internal system database in + // dev/1.3. + !SchemaConstant.SYSTEM_DATABASE.equals( + schemaRegion.getDatabaseFullPath())) + .orElse(false)) .forEach( consensusGroupId -> tmp.put( From c81b0d33d49e5335b37162464483933262f058e0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:26:22 +0800 Subject: [PATCH 2/2] Refine audit series exclusion for dev/1.3 --- .../iotdb/db/schemaengine/SchemaEngine.java | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java index 4784a168768dc..dad19bbdfde22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java @@ -45,6 +45,9 @@ import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionParams; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionLoader; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionParams; +import org.apache.iotdb.db.schemaengine.schemaregion.read.req.SchemaRegionReadPlanFactory; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; @@ -55,6 +58,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -75,6 +79,9 @@ public class SchemaEngine { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final PartialPath AUDIT_LOG_PATH_PATTERN = + new PartialPath(new String[] {"root", "__system", "audit", "**"}); + private final SchemaRegionLoader schemaRegionLoader; @SuppressWarnings("java:S3077") @@ -394,18 +401,56 @@ public Map countTimeSeriesNumBySchemaRegion(final List s .filter( entry -> targetSchemaIds.contains(entry.getKey().getId()) - && SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey()) - // Audit logs are stored under the internal system database in dev/1.3. - && !SchemaConstant.SYSTEM_DATABASE.equals( - entry.getValue().getDatabaseFullPath())) + && SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey())) .forEach( entry -> timeSeriesNum.put( - entry.getKey().getId(), - entry.getValue().getSchemaRegionStatistics().getSeriesNumber(false))); + entry.getKey().getId(), getTimeSeriesNumberForQuota(entry.getValue()))); return timeSeriesNum; } + /** + * Returns the series count used by schema quota. Audit series are stored below {@code + * root.__system.audit} in dev/1.3, while other internal series may share the same system database + * and must remain counted. + */ + private long getTimeSeriesNumberForQuota(final ISchemaRegion schemaRegion) { + final long totalSeriesNumber = schemaRegion.getSchemaRegionStatistics().getSeriesNumber(false); + if (!SchemaConstant.SYSTEM_DATABASE.equals(schemaRegion.getDatabaseFullPath())) { + return totalSeriesNumber; + } + + long auditSeriesNumber = 0; + try { + try (ISchemaReader reader = + schemaRegion.getTimeSeriesReader( + SchemaRegionReadPlanFactory.getShowTimeSeriesPlan( + AUDIT_LOG_PATH_PATTERN, + Collections.emptyMap(), + -1, + 0, + false, + null, + false, + SchemaConstant.ALL_MATCH_SCOPE))) { + while (reader.hasNext()) { + if (!reader.next().isLogicalView()) { + auditSeriesNumber++; + } + } + if (!reader.isSuccess()) { + logger.warn("Failed to count audit time series for schema quota", reader.getFailure()); + return totalSeriesNumber; + } + } + } catch (Exception e) { + // Keep the heartbeat available if the audit-only scan fails; the next heartbeat retries it. + logger.warn("Failed to exclude audit time series from schema quota count", e); + return totalSeriesNumber; + } + return Math.max(0, totalSeriesNumber - auditSeriesNumber); + } + /** * Update total count in schema quota manager and generate local count map response. If limit is * not -1 and deviceNumMap/timeSeriesNumMap is null, fill deviceNumMap/timeSeriesNumMap of the @@ -446,23 +491,13 @@ public void updateAndFillSchemaCountMap(TDataNodeHeartbeatReq req, TDataNodeHear SchemaRegionConsensusImpl.getInstance().getAllConsensusGroupIds().stream() .filter( consensusGroupId -> - SchemaRegionConsensusImpl.getInstance().isLeader(consensusGroupId) - && Optional.ofNullable(schemaRegionMap.get((SchemaRegionId) consensusGroupId)) - .map( - schemaRegion -> - // Audit logs are stored under the internal system database in - // dev/1.3. - !SchemaConstant.SYSTEM_DATABASE.equals( - schemaRegion.getDatabaseFullPath())) - .orElse(false)) + SchemaRegionConsensusImpl.getInstance().isLeader(consensusGroupId)) .forEach( consensusGroupId -> tmp.put( consensusGroupId.getId(), Optional.ofNullable(schemaRegionMap.get(consensusGroupId)) - .map( - schemaRegion -> - schemaRegion.getSchemaRegionStatistics().getSeriesNumber(false)) + .map(this::getTimeSeriesNumberForQuota) .orElse(0L))); } }