diff --git a/docs/commands.md b/docs/commands.md index c84367e60..5f7d52428 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -115,7 +115,7 @@ For example, `/co purge t:30d i:stone,dirt` will delete all stone and dirt data In CoreProtect v2.15+, adding `#optimize` to the end of the command (for example, `/co purge t:30d #optimize`) will also optimize supported database tables and reclaim unused disk space. How this option is handled depends on the database backend: * SQLite already rebuilds the database from retained data and reclaims unused file space as part of a manual purge, so `#optimize` is not needed. -* MySQL normally deletes matching rows. Adding `#optimize` also optimizes its tables to reclaim unused space. +* MySQL normally deletes matching rows. Adding `#optimize` also removes saved entity data that no entity kill references anymore (this requires the `CREATE TEMPORARY TABLES` privilege), then optimizes its tables to reclaim unused space. * DuckDB deletes matching rows in one transaction and checkpoints afterward. `#optimize` has no additional effect. * ClickHouse drops fully covered monthly partitions for an unfiltered time purge and synchronously removes rows from partial or filtered partitions. Adding `#optimize` also runs `OPTIMIZE TABLE ... FINAL`. diff --git a/src/main/java/net/coreprotect/command/PurgeCommand.java b/src/main/java/net/coreprotect/command/PurgeCommand.java index 40a8cdcb3..4164e81f6 100755 --- a/src/main/java/net/coreprotect/command/PurgeCommand.java +++ b/src/main/java/net/coreprotect/command/PurgeCommand.java @@ -28,6 +28,7 @@ import net.coreprotect.config.ConfigHandler; import net.coreprotect.consumer.Consumer; import net.coreprotect.database.Database; +import net.coreprotect.database.PurgeFilter; import net.coreprotect.database.PurgePolicy; import net.coreprotect.language.Phrase; import net.coreprotect.language.Selector; @@ -340,6 +341,7 @@ public void run() { long timeStart = startTime > 0 ? (timestamp - startTime) : 0; long timeEnd = timestamp - endTime; long removed = 0; + PurgeFilter purgeFilter = new PurgeFilter(timeStart, timeEnd, argWid, includeBlockIdsFinal); for (int i = 0; i <= 5; i++) { requirePurgeNotCancelled(); @@ -497,6 +499,9 @@ public void run() { if (table.equals("entity_spawn")) { timeLimit = " WHERE removed=0 OR block_rowid IN(SELECT rowid FROM " + purgePrefix + "block) OR kill_rowid IN(SELECT rowid FROM " + purgePrefix + "entity) OR rowid IN(SELECT entity_spawn_rowid FROM " + purgePrefix + "entity_container) OR rowid IN(SELECT entity_spawn_rowid FROM " + purgePrefix + "entity_interaction)"; } + else if (table.equals("entity")) { + timeLimit = " WHERE " + PurgeFilter.entityRetainCondition(purgePrefix + "block"); + } else if (PurgePolicy.isPurgeable(table)) { String blockRestriction = "("; if (hasBlockRestriction && PurgePolicy.supportsBlockRestriction(table)) { @@ -594,7 +599,13 @@ else if (argWid > 0) { purge = false; } - if (purge) { + if (table.equals("entity")) { + query = PurgeFilter.deleteUnreferencedEntities(purgePrefix + "entity", purgePrefix + "block"); + preparedStmt = preparePurgeStatement(connection, query); + preparedStmt.execute(); + preparedStmt.close(); + } + else if (purge) { query = "DELETE FROM " + purgePrefix + table + " WHERE " + blockRestriction + "time < '" + timeEnd + "' AND time >= '" + timeStart + "'" + worldRestriction; preparedStmt = preparePurgeStatement(connection, query); preparedStmt.execute(); @@ -666,6 +677,13 @@ else if (argWid > 0) { purge = false; } + if (purge && table.equals("block") && purgeFilter.removesKills() && !purgeFilter.purgesEntitiesByTime()) { + query = purgeFilter.deleteEntitiesOfPurgedKills(ConfigHandler.databaseType, ConfigHandler.prefix); + preparedStmt = preparePurgeStatement(connection, query); + removed = removed + preparedStmt.executeUpdate(); + preparedStmt.close(); + } + if (purge) { query = "DELETE FROM " + ConfigHandler.prefix + table + " WHERE " + blockRestriction + "time < '" + timeEnd + "' AND time >= '" + timeStart + "'" + worldRestriction; preparedStmt = preparePurgeStatement(connection, query); @@ -687,6 +705,32 @@ else if (argWid > 0) { } } + if (ConfigHandler.databaseType.isMySQL() && optimize) { + try { + for (String sweepQuery : PurgeFilter.mysqlOrphanSweepSetup(ConfigHandler.prefix)) { + preparedStmt = preparePurgeStatement(connection, sweepQuery); + preparedStmt.execute(); + preparedStmt.close(); + } + preparedStmt = preparePurgeStatement(connection, PurgeFilter.mysqlOrphanSweepDelete(ConfigHandler.prefix)); + removed = removed + preparedStmt.executeUpdate(); + preparedStmt.close(); + } + catch (Exception e) { + reportPurgeFailure(e); + } + finally { + try { + preparedStmt = preparePurgeStatement(connection, PurgeFilter.mysqlOrphanSweepTeardown(ConfigHandler.prefix)); + preparedStmt.execute(); + preparedStmt.close(); + } + catch (Exception e) { + reportPurgeFailure(e); + } + } + } + requirePurgeNotCancelled(); String retainedPrefix = ConfigHandler.databaseType.isSQLite() ? purgePrefix : ConfigHandler.prefix; query = "UPDATE " + retainedPrefix + "entity_spawn SET kill_rowid=NULL WHERE kill_rowid IS NOT NULL AND NOT EXISTS (SELECT 1 FROM " + retainedPrefix + "entity WHERE " + retainedPrefix + "entity.rowid=" + retainedPrefix + "entity_spawn.kill_rowid)"; diff --git a/src/main/java/net/coreprotect/database/PurgeFilter.java b/src/main/java/net/coreprotect/database/PurgeFilter.java new file mode 100644 index 000000000..5879cec89 --- /dev/null +++ b/src/main/java/net/coreprotect/database/PurgeFilter.java @@ -0,0 +1,185 @@ +package net.coreprotect.database; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; + +import net.coreprotect.model.action.LookupActions; + +/** + * Selects the rows that one /co purge removes on the relational backends (SQLite, MySQL, DuckDB). + * + *

+ * co_entity has no world or type column: each row belongs to the co_block kill row whose data column holds its + * rowid. The statements built here remove co_entity rows through those kill rows, so a scoped purge never leaves + * entity data behind without the kill row that references it. Player kills also use the kill action, but they + * store type 0 and a user id in data, so they are never treated as co_entity references. + */ +public final class PurgeFilter { + + private final long timeStart; + private final long timeEnd; + private final int worldId; + private final List blockTypes; + + /** + * @param timeStart + * the oldest time (inclusive) to purge + * @param timeEnd + * the newest time (exclusive) to purge + * @param worldId + * the world to purge, or 0 for every world + * @param blockTypes + * material ids (i:) that restrict the purge to those co_block rows, or an empty list + */ + public PurgeFilter(long timeStart, long timeEnd, int worldId, List blockTypes) { + this.timeStart = timeStart; + this.timeEnd = timeEnd; + this.worldId = worldId; + this.blockTypes = List.copyOf(blockTypes); + } + + /** + * Returns whether the purge is limited to selected co_block rows, which leaves the other tables untouched. + */ + private boolean restrictsTables() { + return !blockTypes.isEmpty(); + } + + /** + * Returns whether the purge removes co_block kill rows. A restriction to materials keeps every kill row. + */ + public boolean removesKills() { + return blockTypes.isEmpty(); + } + + /** + * Returns whether co_entity rows can be removed by time alone. This holds when every kill row in the time range + * is purged, because a co_entity row always has the same time as its kill row. + */ + public boolean purgesEntitiesByTime() { + return worldId <= 0 && !restrictsTables(); + } + + /** + * Builds the condition that matches the co_block rows this purge removes. + * + * @param qualifier + * the table alias to prefix each column with, such as "b.", or an empty string + * @return the SQL condition + */ + private String blockCondition(String qualifier) { + StringBuilder condition = new StringBuilder(timeCondition(qualifier)); + if (worldId > 0) { + condition.append(" AND ").append(qualifier).append("wid = ").append(worldId); + } + + if (!blockTypes.isEmpty()) { + condition.append(" AND ").append(qualifier).append("action NOT IN(").append(LookupActions.ENTITY_KILL).append(",").append(LookupActions.ENTITY_SPAWN).append(")"); + condition.append(" AND ").append(qualifier).append("type IN(").append(joinIds(blockTypes)).append(")"); + } + return condition.toString(); + } + + /** + * Builds the MySQL or DuckDB statement that deletes the co_entity rows of the kill rows this purge removes. Run it + * before the co_block delete, while those kill rows still exist. + * + * @param databaseType + * the MySQL or DuckDB backend + * @param prefix + * the table prefix + * @return the SQL statement + */ + public String deleteEntitiesOfPurgedKills(DatabaseType databaseType, String prefix) { + if (databaseType.isMySQL()) { + // The join form avoids a dependent subquery on MySQL 5.7 and MariaDB. + return "DELETE e FROM " + prefix + "entity AS e INNER JOIN " + prefix + "block AS b ON b.data = e.rowid WHERE " + killReference("b.") + " AND " + blockCondition("b."); + } + return "DELETE FROM " + prefix + "entity WHERE rowid IN(SELECT data FROM " + prefix + "block WHERE " + killReference("") + " AND " + blockCondition("") + ")"; + } + + /** + * Builds the SQLite copy condition that keeps the co_entity rows still referenced by a retained kill row. + * + * @param retainedBlockTable + * the co_block table that holds the rows the purge keeps + * @return the SQL condition + */ + public static String entityRetainCondition(String retainedBlockTable) { + return "rowid IN(" + killReferences(retainedBlockTable) + ")"; + } + + /** + * Builds the statement that deletes co_entity rows that no kill row references. + * + * @param entityTable + * the co_entity table to clean + * @param blockTable + * the co_block table that holds the kill rows + * @return the SQL statement + */ + public static String deleteUnreferencedEntities(String entityTable, String blockTable) { + return "DELETE FROM " + entityTable + " WHERE rowid NOT IN(" + killReferences(blockTable) + ")"; + } + + /** + * Builds the MySQL statements that prepare an orphan sweep: a temporary table of the co_entity ids that kill rows + * reference. The table avoids an anti-join on the unindexed co_block.data column. + * + * @param prefix + * the table prefix + * @return the SQL statements, in order + */ + public static List mysqlOrphanSweepSetup(String prefix) { + String keepTable = prefix + "entity_keep"; + List statements = new ArrayList<>(); + statements.add("DROP TEMPORARY TABLE IF EXISTS " + keepTable); + statements.add("CREATE TEMPORARY TABLE " + keepTable + " (rowid INT NOT NULL PRIMARY KEY) ENGINE=InnoDB"); + statements.add("INSERT IGNORE INTO " + keepTable + " (rowid) " + killReferences(prefix + "block")); + return statements; + } + + /** + * Builds the MySQL statement that deletes co_entity rows no kill row references, such as rows left by earlier + * world purges. Run {@link #mysqlOrphanSweepSetup(String)} first. + * + * @param prefix + * the table prefix + * @return the SQL statement + */ + public static String mysqlOrphanSweepDelete(String prefix) { + return "DELETE e FROM " + prefix + "entity AS e LEFT JOIN " + prefix + "entity_keep AS k ON k.rowid = e.rowid WHERE k.rowid IS NULL"; + } + + /** + * Builds the MySQL statement that drops the temporary table of an orphan sweep. + * + * @param prefix + * the table prefix + * @return the SQL statement + */ + public static String mysqlOrphanSweepTeardown(String prefix) { + return "DROP TEMPORARY TABLE IF EXISTS " + prefix + "entity_keep"; + } + + private String timeCondition(String qualifier) { + return qualifier + "time < " + timeEnd + " AND " + qualifier + "time >= " + timeStart; + } + + private static String killReferences(String blockTable) { + return "SELECT data FROM " + blockTable + " WHERE " + killReference("") + " AND data IS NOT NULL"; + } + + private static String killReference(String qualifier) { + return qualifier + "action = " + LookupActions.ENTITY_KILL + " AND " + qualifier + "type <> 0"; + } + + private static String joinIds(List ids) { + StringJoiner joiner = new StringJoiner(","); + for (Integer id : ids) { + joiner.add(String.valueOf(id)); + } + return joiner.toString(); + } +}