Fix orphaned entity data

docs/commands.md

@@ -115,7 +115,7 @@ For example, `/co purge t:30d i:stone,dirt` will delete all stone and dirt data
 In CoreProtect v2.15+, adding `#optimize` to the end of the command (for example, `/co purge t:30d #optimize`) will also optimize supported database tables and reclaim unused disk space. How this option is handled depends on the database backend:
 
 * SQLite already rebuilds the database from retained data and reclaims unused file space as part of a manual purge, so `#optimize` is not needed.
-* MySQL normally deletes matching rows. Adding `#optimize` also optimizes its tables to reclaim unused space.
+* MySQL normally deletes matching rows. Adding `#optimize` also removes saved entity data that no entity kill references anymore (this requires the `CREATE TEMPORARY TABLES` privilege), then optimizes its tables to reclaim unused space.
 * DuckDB deletes matching rows in one transaction and checkpoints afterward. `#optimize` has no additional effect.
 * ClickHouse drops fully covered monthly partitions for an unfiltered time purge and synchronously removes rows from partial or filtered partitions. Adding `#optimize` also runs `OPTIMIZE TABLE ... FINAL`.
 

src/main/java/net/coreprotect/command/PurgeCommand.java

@@ -28,6 +28,7 @@
 import net.coreprotect.config.ConfigHandler;
 import net.coreprotect.consumer.Consumer;
 import net.coreprotect.database.Database;
+import net.coreprotect.database.PurgeFilter;
 import net.coreprotect.database.PurgePolicy;
 import net.coreprotect.language.Phrase;
 import net.coreprotect.language.Selector;
@@ -340,6 +341,7 @@ public void run() {
                     long timeStart = startTime > 0 ? (timestamp - startTime) : 0;
                     long timeEnd = timestamp - endTime;
                     long removed = 0;
+                    PurgeFilter purgeFilter = new PurgeFilter(timeStart, timeEnd, argWid, includeBlockIdsFinal);
 
                     for (int i = 0; i <= 5; i++) {
                         requirePurgeNotCancelled();
@@ -497,6 +499,9 @@ public void run() {
                                     if (table.equals("entity_spawn")) {
                                         timeLimit = " WHERE removed=0 OR block_rowid IN(SELECT rowid FROM " + purgePrefix + "block) OR kill_rowid IN(SELECT rowid FROM " + purgePrefix + "entity) OR rowid IN(SELECT entity_spawn_rowid FROM " + purgePrefix + "entity_container) OR rowid IN(SELECT entity_spawn_rowid FROM " + purgePrefix + "entity_interaction)";
                                     }
+                                    else if (table.equals("entity")) {
+                                        timeLimit = " WHERE " + PurgeFilter.entityRetainCondition(purgePrefix + "block");
+                                    }
                                     else if (PurgePolicy.isPurgeable(table)) {
                                         String blockRestriction = "(";
                                         if (hasBlockRestriction && PurgePolicy.supportsBlockRestriction(table)) {
@@ -594,7 +599,13 @@ else if (argWid > 0) {
                                         purge = false;
                                     }
 
-                                    if (purge) {
+                                    if (table.equals("entity")) {
+                                        query = PurgeFilter.deleteUnreferencedEntities(purgePrefix + "entity", purgePrefix + "block");
+                                        preparedStmt = preparePurgeStatement(connection, query);
+                                        preparedStmt.execute();
+                                        preparedStmt.close();
+                                    }
+                                    else if (purge) {
                                         query = "DELETE FROM " + purgePrefix + table + " WHERE " + blockRestriction + "time < '" + timeEnd + "' AND time >= '" + timeStart + "'" + worldRestriction;
                                         preparedStmt = preparePurgeStatement(connection, query);
                                         preparedStmt.execute();
@@ -666,6 +677,13 @@ else if (argWid > 0) {
                                     purge = false;
                                 }
 
+                                if (purge && table.equals("block") && purgeFilter.removesKills() && !purgeFilter.purgesEntitiesByTime()) {
+                                    query = purgeFilter.deleteEntitiesOfPurgedKills(ConfigHandler.databaseType, ConfigHandler.prefix);
+                                    preparedStmt = preparePurgeStatement(connection, query);
+                                    removed = removed + preparedStmt.executeUpdate();
+                                    preparedStmt.close();
+                                }
+
                                 if (purge) {
                                     query = "DELETE FROM " + ConfigHandler.prefix + table + " WHERE " + blockRestriction + "time < '" + timeEnd + "' AND time >= '" + timeStart + "'" + worldRestriction;
                                     preparedStmt = preparePurgeStatement(connection, query);
@@ -687,6 +705,32 @@ else if (argWid > 0) {
                         }
                     }
 
+                    if (ConfigHandler.databaseType.isMySQL() && optimize) {
+                        try {
+                            for (String sweepQuery : PurgeFilter.mysqlOrphanSweepSetup(ConfigHandler.prefix)) {
+                                preparedStmt = preparePurgeStatement(connection, sweepQuery);
+                                preparedStmt.execute();
+                                preparedStmt.close();
+                            }
+                            preparedStmt = preparePurgeStatement(connection, PurgeFilter.mysqlOrphanSweepDelete(ConfigHandler.prefix));
+                            removed = removed + preparedStmt.executeUpdate();
+                            preparedStmt.close();
+                        }
+                        catch (Exception e) {
+                            reportPurgeFailure(e);
+                        }
+                        finally {
+                            try {
+                                preparedStmt = preparePurgeStatement(connection, PurgeFilter.mysqlOrphanSweepTeardown(ConfigHandler.prefix));
+                                preparedStmt.execute();
+                                preparedStmt.close();
+                            }
+                            catch (Exception e) {
+                                reportPurgeFailure(e);
+                            }
+                        }
+                    }
+
                     requirePurgeNotCancelled();
                     String retainedPrefix = ConfigHandler.databaseType.isSQLite() ? purgePrefix : ConfigHandler.prefix;
                     query = "UPDATE " + retainedPrefix + "entity_spawn SET kill_rowid=NULL WHERE kill_rowid IS NOT NULL AND NOT EXISTS (SELECT 1 FROM " + retainedPrefix + "entity WHERE " + retainedPrefix + "entity.rowid=" + retainedPrefix + "entity_spawn.kill_rowid)";

src/main/java/net/coreprotect/database/PurgeFilter.java

@@ -0,0 +1,185 @@
+package net.coreprotect.database;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import net.coreprotect.model.action.LookupActions;
+
+/**
+ * Selects the rows that one /co purge removes on the relational backends (SQLite, MySQL, DuckDB).
+ *
+ * <p>
+ * co_entity has no world or type column: each row belongs to the co_block kill row whose data column holds its
+ * rowid. The statements built here remove co_entity rows through those kill rows, so a scoped purge never leaves
+ * entity data behind without the kill row that references it. Player kills also use the kill action, but they
+ * store type 0 and a user id in data, so they are never treated as co_entity references.
+ */
+public final class PurgeFilter {
+
+    private final long timeStart;
+    private final long timeEnd;
+    private final int worldId;
+    private final List<Integer> blockTypes;
+
+    /**
+     * @param timeStart
+     *            the oldest time (inclusive) to purge
+     * @param timeEnd
+     *            the newest time (exclusive) to purge
+     * @param worldId
+     *            the world to purge, or 0 for every world
+     * @param blockTypes
+     *            material ids (i:) that restrict the purge to those co_block rows, or an empty list
+     */
+    public PurgeFilter(long timeStart, long timeEnd, int worldId, List<Integer> blockTypes) {
+        this.timeStart = timeStart;
+        this.timeEnd = timeEnd;
+        this.worldId = worldId;
+        this.blockTypes = List.copyOf(blockTypes);
+    }
+
+    /**
+     * Returns whether the purge is limited to selected co_block rows, which leaves the other tables untouched.
+     */
+    private boolean restrictsTables() {
+        return !blockTypes.isEmpty();
+    }
+
+    /**
+     * Returns whether the purge removes co_block kill rows. A restriction to materials keeps every kill row.
+     */
+    public boolean removesKills() {
+        return blockTypes.isEmpty();
+    }
+
+    /**
+     * Returns whether co_entity rows can be removed by time alone. This holds when every kill row in the time range
+     * is purged, because a co_entity row always has the same time as its kill row.
+     */
+    public boolean purgesEntitiesByTime() {
+        return worldId <= 0 && !restrictsTables();
+    }
+
+    /**
+     * Builds the condition that matches the co_block rows this purge removes.
+     *
+     * @param qualifier
+     *            the table alias to prefix each column with, such as "b.", or an empty string
+     * @return the SQL condition
+     */
+    private String blockCondition(String qualifier) {
+        StringBuilder condition = new StringBuilder(timeCondition(qualifier));
+        if (worldId > 0) {
+            condition.append(" AND ").append(qualifier).append("wid = ").append(worldId);
+        }
+
+        if (!blockTypes.isEmpty()) {
+            condition.append(" AND ").append(qualifier).append("action NOT IN(").append(LookupActions.ENTITY_KILL).append(",").append(LookupActions.ENTITY_SPAWN).append(")");
+            condition.append(" AND ").append(qualifier).append("type IN(").append(joinIds(blockTypes)).append(")");
+        }
+        return condition.toString();
+    }
+
+    /**
+     * Builds the MySQL or DuckDB statement that deletes the co_entity rows of the kill rows this purge removes. Run it
+     * before the co_block delete, while those kill rows still exist.
+     *
+     * @param databaseType
+     *            the MySQL or DuckDB backend
+     * @param prefix
+     *            the table prefix
+     * @return the SQL statement
+     */
+    public String deleteEntitiesOfPurgedKills(DatabaseType databaseType, String prefix) {
+        if (databaseType.isMySQL()) {
+            // The join form avoids a dependent subquery on MySQL 5.7 and MariaDB.
+            return "DELETE e FROM " + prefix + "entity AS e INNER JOIN " + prefix + "block AS b ON b.data = e.rowid WHERE " + killReference("b.") + " AND " + blockCondition("b.");
+        }
+        return "DELETE FROM " + prefix + "entity WHERE rowid IN(SELECT data FROM " + prefix + "block WHERE " + killReference("") + " AND " + blockCondition("") + ")";
+    }
+
+    /**
+     * Builds the SQLite copy condition that keeps the co_entity rows still referenced by a retained kill row.
+     *
+     * @param retainedBlockTable
+     *            the co_block table that holds the rows the purge keeps
+     * @return the SQL condition
+     */
+    public static String entityRetainCondition(String retainedBlockTable) {
+        return "rowid IN(" + killReferences(retainedBlockTable) + ")";
+    }
+
+    /**
+     * Builds the statement that deletes co_entity rows that no kill row references.
+     *
+     * @param entityTable
+     *            the co_entity table to clean
+     * @param blockTable
+     *            the co_block table that holds the kill rows
+     * @return the SQL statement
+     */
+    public static String deleteUnreferencedEntities(String entityTable, String blockTable) {
+        return "DELETE FROM " + entityTable + " WHERE rowid NOT IN(" + killReferences(blockTable) + ")";
+    }
+
+    /**
+     * Builds the MySQL statements that prepare an orphan sweep: a temporary table of the co_entity ids that kill rows
+     * reference. The table avoids an anti-join on the unindexed co_block.data column.
+     *
+     * @param prefix
+     *            the table prefix
+     * @return the SQL statements, in order
+     */
+    public static List<String> mysqlOrphanSweepSetup(String prefix) {
+        String keepTable = prefix + "entity_keep";
+        List<String> statements = new ArrayList<>();
+        statements.add("DROP TEMPORARY TABLE IF EXISTS " + keepTable);
+        statements.add("CREATE TEMPORARY TABLE " + keepTable + " (rowid INT NOT NULL PRIMARY KEY) ENGINE=InnoDB");
+        statements.add("INSERT IGNORE INTO " + keepTable + " (rowid) " + killReferences(prefix + "block"));
+        return statements;
+    }
+
+    /**
+     * Builds the MySQL statement that deletes co_entity rows no kill row references, such as rows left by earlier
+     * world purges. Run {@link #mysqlOrphanSweepSetup(String)} first.
+     *
+     * @param prefix
+     *            the table prefix
+     * @return the SQL statement
+     */
+    public static String mysqlOrphanSweepDelete(String prefix) {
+        return "DELETE e FROM " + prefix + "entity AS e LEFT JOIN " + prefix + "entity_keep AS k ON k.rowid = e.rowid WHERE k.rowid IS NULL";
+    }
+
+    /**
+     * Builds the MySQL statement that drops the temporary table of an orphan sweep.
+     *
+     * @param prefix
+     *            the table prefix
+     * @return the SQL statement
+     */
+    public static String mysqlOrphanSweepTeardown(String prefix) {
+        return "DROP TEMPORARY TABLE IF EXISTS " + prefix + "entity_keep";
+    }
+
+    private String timeCondition(String qualifier) {
+        return qualifier + "time < " + timeEnd + " AND " + qualifier + "time >= " + timeStart;
+    }
+
+    private static String killReferences(String blockTable) {
+        return "SELECT data FROM " + blockTable + " WHERE " + killReference("") + " AND data IS NOT NULL";
+    }
+
+    private static String killReference(String qualifier) {
+        return qualifier + "action = " + LookupActions.ENTITY_KILL + " AND " + qualifier + "type <> 0";
+    }
+
+    private static String joinIds(List<Integer> ids) {
+        StringJoiner joiner = new StringJoiner(",");
+        for (Integer id : ids) {
+            joiner.add(String.valueOf(id));
+        }
+        return joiner.toString();
+    }
+}