From 1513fe3d4a8c0540c7f8faf316d937d3b504d467 Mon Sep 17 00:00:00 2001 From: Daniil Date: Mon, 29 Dec 2025 15:41:37 +0300 Subject: [PATCH] add real size reporting to idle_verify command --- .../GridCommandHandlerClusterByClassTest.java | 2 +- .../cache/VerifyBackupPartitionsDumpTask.java | 4 +- .../snapshot/IncrementalSnapshotVerify.java | 1 + .../SnapshotPartitionsQuickVerifyHandler.java | 3 +- .../SnapshotPartitionsVerifyHandler.java | 15 ++++-- .../cache/verify/IdleVerifyUtility.java | 7 +++ .../cache/verify/PartitionHashRecord.java | 49 +++++++++++++------ 7 files changed, 58 insertions(+), 23 deletions(-) diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java index efc11f3cd5bdd..aa32e24465e6f 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerClusterByClassTest.java @@ -753,7 +753,7 @@ public void testCacheIdleVerifyDump() throws Exception { assertContains(log, dumpWithZeros, "The check procedure has finished, found " + parts + " partitions"); assertContains(log, dumpWithZeros, "Partition: PartitionKey [grpId=1544803905, grpName=default, partId=0]"); - assertContains(log, dumpWithZeros, "updateCntr=0, partitionState=OWNING, size=0, partHash=0"); + assertContains(log, dumpWithZeros, "updateCntr=0, partitionState=OWNING, metaSize=0, realSize=0, partHash=0"); assertContains(log, dumpWithZeros, "no conflicts have been found"); assertCompactFooterStat(dumpWithZeros, 0, 0, 0, keysCnt); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java index 8606b93e79ac4..bfad8dcf6592a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/management/cache/VerifyBackupPartitionsDumpTask.java @@ -144,7 +144,7 @@ private boolean needToAdd(List records) { PartitionHashRecord record = records.get(0); - if (record.size() != 0) + if (record.realSize() != 0) return true; int firstHash = record.partitionHash(); @@ -155,7 +155,7 @@ record = records.get(i); if (record.partitionHash() != firstHash || record.partitionVersionsHash() != firstVerHash - || record.size() != 0 + || record.realSize() != 0 ) return true; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerify.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerify.java index a206415a498d6..d2790a2179d15 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerify.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IncrementalSnapshotVerify.java @@ -254,6 +254,7 @@ else if (txRec.state() == TransactionState.ROLLED_BACK) { sft.consistentId(), null, 0, + 0, null, new VerifyPartitionContext(e.getValue()) ) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java index 84f8b90311f5c..a1609e56cf0ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsQuickVerifyHandler.java @@ -88,7 +88,8 @@ public SnapshotPartitionsQuickVerifyHandler(GridCacheSharedContext cctx) { PartitionHashRecord other = total.putIfAbsent(part, val); if ((other != null && !wrnGrps.contains(part.groupId())) - && ((!val.hasExpiringEntries() && !other.hasExpiringEntries() && val.size() != other.size()) + && ((!val.hasExpiringEntries() && !other.hasExpiringEntries() + && (val.metaSize() != other.metaSize() || val.realSize() != other.realSize())) || !Objects.equals(val.updateCounter(), other.updateCounter()))) wrnGrps.add(part.groupId()); }); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java index c12f86e15efd7..a34420da8d2d9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyHandler.java @@ -403,6 +403,7 @@ private PartitionHashRecord calculateDumpedPartitionHash(Dump dump, String folde cctx.localNode().consistentId(), null, 0, + 0, PartitionHashRecord.PartitionState.OWNING, new VerifyPartitionContext() ); @@ -410,16 +411,19 @@ private PartitionHashRecord calculateDumpedPartitionHash(Dump dump, String folde try { try (Dump.DumpedPartitionIterator iter = dump.iterator(folderName, CU.cacheId(grpName), part, null)) { - long size = 0; + long totalSize = 0; + long realSize = 0; VerifyPartitionContext ctx = new VerifyPartitionContext(); while (iter.hasNext()) { DumpEntry e = iter.next(); + totalSize++; - ctx.update((KeyCacheObject)e.key(), (CacheObject)e.value(), e.version()); - - size++; + if (e.expireTime() <= 0) { + ctx.update((KeyCacheObject)e.key(), (CacheObject)e.value(), e.version()); + realSize++; + } } return new PartitionHashRecord( @@ -427,7 +431,8 @@ private PartitionHashRecord calculateDumpedPartitionHash(Dump dump, String folde false, cctx.localNode().consistentId(), null, - size, + totalSize, + realSize, PartitionHashRecord.PartitionState.OWNING, ctx ); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java index 1dc7c5df03e3f..71dda39470e7b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java @@ -298,6 +298,7 @@ public static List compareUpdateCounters( updCntr, state == GridDhtPartitionState.MOVING ? PartitionHashRecord.MOVING_PARTITION_SIZE : 0, + 0, state == GridDhtPartitionState.MOVING ? PartitionHashRecord.PartitionState.MOVING : PartitionHashRecord.PartitionState.LOST, new VerifyPartitionContext() @@ -327,6 +328,7 @@ public static List compareUpdateCounters( consId, updCntr, partSize, + ctx.entryCount, PartitionHashRecord.PartitionState.OWNING, ctx ); @@ -391,6 +393,9 @@ public static class VerifyPartitionContext { /** */ public int regular; + /** */ + public int entryCount; + /** */ public VerifyPartitionContext() { // No-op. @@ -410,6 +415,8 @@ public void update( CacheObject val, CacheEntryVersion ver ) throws IgniteCheckedException { + entryCount++; + partHash += key.hashCode(); if (ver != null) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java index 14c68ef648f0d..aae091f1aaec9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecord.java @@ -64,9 +64,13 @@ public class PartitionHashRecord extends IgniteDataTransferObject { @GridToStringInclude private Object updateCntr; - /** Size. */ + /** Size from partition metadata. */ @GridToStringExclude - private long size; + private long metaSize; + + /** Actual number of entries in partition. */ + @GridToStringExclude + private long realSize; /** Partition state. */ private PartitionState partitionState; @@ -105,7 +109,8 @@ public class PartitionHashRecord extends IgniteDataTransferObject { * @param isPrimary Is primary. * @param consistentId Consistent id. * @param updateCntr Update counter. - * @param size Size. + * @param metaSize Size from partition metadata. + * @param realSize Actual number of entries in partition. * @param partitionState Partition state. * @param ctx Verify partition data. */ @@ -114,7 +119,8 @@ public PartitionHashRecord( boolean isPrimary, Object consistentId, Object updateCntr, - long size, + long metaSize, + long realSize, PartitionState partitionState, VerifyPartitionContext ctx ) { @@ -124,7 +130,8 @@ public PartitionHashRecord( this.partHash = ctx.partHash; this.partVerHash = ctx.partVerHash; this.updateCntr = updateCntr; - this.size = size; + this.metaSize = metaSize; + this.realSize = realSize; this.partitionState = partitionState; this.cfKeys = ctx.cf; this.noCfKeys = ctx.noCf; @@ -181,10 +188,17 @@ public Object updateCounter() { } /** - * @return Size. + * @return Size from metadata. + */ + public long metaSize() { + return metaSize; + } + + /** + * @return Actual number of entries in partition. */ - public long size() { - return size; + public long realSize() { + return realSize; } /** @@ -232,7 +246,8 @@ public void hasExpiringEntries(boolean hasExpiringEntries) { out.writeInt(partHash); out.writeInt(partVerHash); out.writeObject(updateCntr); - out.writeLong(size); + out.writeLong(metaSize); + out.writeLong(realSize); U.writeEnum(out, partitionState); out.writeInt(cfKeys); out.writeInt(noCfKeys); @@ -249,7 +264,8 @@ public void hasExpiringEntries(boolean hasExpiringEntries) { partHash = in.readInt(); partVerHash = in.readInt(); updateCntr = in.readObject(); - size = in.readLong(); + metaSize = in.readLong(); + realSize = in.readLong(); partitionState = PartitionState.fromOrdinal(in.readByte()); cfKeys = in.readInt(); noCfKeys = in.readInt(); @@ -260,9 +276,13 @@ public void hasExpiringEntries(boolean hasExpiringEntries) { /** {@inheritDoc} */ @Override public String toString() { - return size == MOVING_PARTITION_SIZE ? + return metaSize == MOVING_PARTITION_SIZE ? S.toString(PartitionHashRecord.class, this, "state", "MOVING") : - S.toString(PartitionHashRecord.class, this, "size", size, "partHash", partHash, "partVerHash", partVerHash); + S.toString(PartitionHashRecord.class, this, + "metaSize", metaSize, + "realSize", realSize, + "partHash", partHash, + "partVerHash", partVerHash); } /** {@inheritDoc} */ @@ -276,7 +296,8 @@ public void hasExpiringEntries(boolean hasExpiringEntries) { PartitionHashRecord v2 = (PartitionHashRecord)o; return partHash == v2.partHash && partVerHash == v2.partVerHash && Objects.equals(updateCntr, v2.updateCntr) && - size == v2.size && partKey.equals(v2.partKey) && consistentId.equals(v2.consistentId) && + metaSize == v2.metaSize && realSize == v2.realSize && + partKey.equals(v2.partKey) && consistentId.equals(v2.consistentId) && partitionState == v2.partitionState && cfKeys == v2.cfKeys && noCfKeys == v2.noCfKeys && binKeys == v2.binKeys && regKeys == v2.regKeys; @@ -284,7 +305,7 @@ public void hasExpiringEntries(boolean hasExpiringEntries) { /** {@inheritDoc} */ @Override public int hashCode() { - return Objects.hash(partKey, consistentId, partHash, partVerHash, updateCntr, size, partitionState, + return Objects.hash(partKey, consistentId, partHash, partVerHash, updateCntr, metaSize, realSize, partitionState, cfKeys, noCfKeys, binKeys, regKeys); }